From 8ee9231d49baf4806445be4e17c7ef56e9623f02 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 8 Nov 2026 14:16:21 -0800 Subject: [PATCH] perf(profile-state): store large domains as per-entry rows so saves rewrite only what changed (#39974) Session, remote layout and worktree metadata domains are stored one row per entry, so a save rewrites only the entries that changed instead of the whole multi-megabyte document. Readers reassemble the exact original JSON. The schema version is unchanged, so older builds keep reading the full-document row, which is refreshed when the writer starts, at most every five minutes while it runs (including quiet periods), at graceful close, in published backups, and after profile transfers. --- config/main-layout-writer-baseline.txt | 2 +- .../profile-project-domain-state.ts | 3 + .../workspace-session-write-listeners.ts | 8 +- .../profile-state-database-schema.ts | 23 ++ .../profile-state-database-snapshot.ts | 4 + .../profile-state/profile-state-database.ts | 3 + .../profile-state-document-part-ordinals.ts | 73 +++++ .../profile-state-document-parts-reader.ts | 76 +++++ .../profile-state-document-parts-storage.ts | 303 ++++++++++++++++++ .../profile-state-document-parts.test.ts | 168 ++++++++++ .../profile-state-document-parts.ts | 150 +++++++++ .../profile-state-document-reader.ts | 11 +- .../profile-state/profile-state-documents.ts | 3 + .../profile-state-domain-reader.ts | 13 +- .../profile-state-domain-write-validation.ts | 6 +- .../profile-state-domain-writes.ts | 49 +++ .../profile-state-legacy-document-refresh.ts | 137 ++++++++ .../profile-state/profile-state-revision.ts | 11 +- .../profile-state-section-repair.ts | 5 +- .../profile-state-sqlite-authority.ts | 11 +- .../profile-state-worker-authority.test.ts | 33 ++ .../profile-state-worker-authority.ts | 19 +- .../profile-state-writer-connection.ts | 8 +- .../profile-state-writer-protocol.ts | 10 +- .../profile-state-writer-worker-entry.ts | 2 +- .../persistence/workspace-session-domains.ts | 2 + 26 files changed, 1095 insertions(+), 38 deletions(-) create mode 100644 src/main/persistence/profile-state/profile-state-document-part-ordinals.ts create mode 100644 src/main/persistence/profile-state/profile-state-document-parts-reader.ts create mode 100644 src/main/persistence/profile-state/profile-state-document-parts-storage.ts create mode 100644 src/main/persistence/profile-state/profile-state-document-parts.test.ts create mode 100644 src/main/persistence/profile-state/profile-state-document-parts.ts create mode 100644 src/main/persistence/profile-state/profile-state-legacy-document-refresh.ts create mode 100644 src/main/persistence/workspace-session-domains.ts diff --git a/config/main-layout-writer-baseline.txt b/config/main-layout-writer-baseline.txt index 639a055288a..096973f7717 100644 --- a/config/main-layout-writer-baseline.txt +++ b/config/main-layout-writer-baseline.txt @@ -29,7 +29,6 @@ 3 src/main/persistence/loading-store/state-serialization-secret-handling.ts 4 src/main/persistence/loading-store/workspace-session-snapshot-publication.ts 4 src/main/persistence/loading-store/workspace-session-terminal-binding-replay.ts -2 src/main/persistence/loading-store/workspace-session-write-listeners.ts 2 src/main/persistence/migrating-orcad-catalog/orcad-destination-client-state.ts 1 src/main/persistence/migrating-orcad-catalog/orcad-destination-workspace-session.ts 3 src/main/persistence/migrating-orcad-catalog/orcad-dormant-state-records.ts @@ -50,6 +49,7 @@ 1 src/main/persistence/tracking-repos/project-group-operations.ts 4 src/main/persistence/tracking-repos/repo-worktree-pruning.ts 3 src/main/persistence/tracking-repos/worktree-identity-migration.ts +2 src/main/persistence/workspace-session-domains.ts 2 src/main/pty/scrollback/terminal-scrollback-snapshots.ts 1 src/main/runtime/browser/client-pages/client-hosted-browser-page-persistence.ts 1 src/main/runtime/client/runtime-client-state.ts diff --git a/src/main/orca-profiles/profile-project-domain-state.ts b/src/main/orca-profiles/profile-project-domain-state.ts index 1ac7ebc9893..d691ee0e1f6 100644 --- a/src/main/orca-profiles/profile-project-domain-state.ts +++ b/src/main/orca-profiles/profile-project-domain-state.ts @@ -1,3 +1,4 @@ +import { ProfileStateLegacyDocumentRefresh } from '../persistence/profile-state/profile-state-legacy-document-refresh' import { openProfileStateDatabase, openProfileStateDatabaseReadOnly @@ -78,6 +79,8 @@ export function writeProfileProjectDomainChanges( if (!result.changed || result.revision !== changes.expectedRevision + 1) { throw new Error('Profile domain transfer did not commit its expected revision') } + // This profile may not be running; keep the copy older builds read current too. + new ProfileStateLegacyDocumentRefresh().refresh(opened.db) } finally { opened.db.close() } diff --git a/src/main/persistence/loading-store/workspace-session-write-listeners.ts b/src/main/persistence/loading-store/workspace-session-write-listeners.ts index c1772e4229b..87e66bf49eb 100644 --- a/src/main/persistence/loading-store/workspace-session-write-listeners.ts +++ b/src/main/persistence/loading-store/workspace-session-write-listeners.ts @@ -2,6 +2,7 @@ import type { ExecutionHostId } from '../../../shared/execution-host' import { ORCA_LAYOUT_OWNER_IS_MAIN } from '../../../shared/layout-owner-switch' import type { StoreRuntimeState } from './store-runtime-state' import { refileEditorViews } from './layout-editor-views' +import { WORKSPACE_SESSION_DOMAINS } from '../workspace-session-domains' /** Null means any partition may have changed. */ export type WorkspaceSessionWriteListener = (hostId: ExecutionHostId | null) => void @@ -9,10 +10,7 @@ export type WorkspaceSessionWriteListener = (hostId: ExecutionHostId | null) => /** Null means any partition may have changed. */ export type WorkspaceSessionOperationListener = (hostIds: readonly ExecutionHostId[] | null) => void -const WORKSPACE_SESSION_DOMAINS: readonly string[] = [ - 'workspaceSession', - 'workspaceSessionsByHostId' -] +const SESSION_DOMAINS: readonly string[] = WORKSPACE_SESSION_DOMAINS /** Full saves may contain untracked session edits; an absent host makes all partitions candidates. */ export function notifyWorkspaceSessionWritten( @@ -23,7 +21,7 @@ export function notifyWorkspaceSessionWritten( dirtyDomains?: readonly string[], hostId?: ExecutionHostId ): void { - if (dirtyDomains && !dirtyDomains.some((domain) => WORKSPACE_SESSION_DOMAINS.includes(domain))) { + if (dirtyDomains && !dirtyDomains.some((domain) => SESSION_DOMAINS.includes(domain))) { return } // Refile views before listeners project the updated session. diff --git a/src/main/persistence/profile-state/profile-state-database-schema.ts b/src/main/persistence/profile-state/profile-state-database-schema.ts index 2b66285478b..9b4a0ff4f39 100644 --- a/src/main/persistence/profile-state/profile-state-database-schema.ts +++ b/src/main/persistence/profile-state/profile-state-database-schema.ts @@ -38,3 +38,26 @@ export function createProfileStateTablesSql(): string { updated_at INTEGER NOT NULL );` } + +/** Additive split-domain storage (see profile-state-document-parts); older v3 builds ignore it. */ +export const PROFILE_STATE_DOCUMENT_PARTS_TABLE = 'profile_state_document_parts' +export const PROFILE_STATE_DOCUMENT_PARTS_META_TABLE = 'profile_state_document_parts_meta' + +export function createProfileStateDocumentPartsTablesSql(): string { + return `CREATE TABLE IF NOT EXISTS ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE} ( + domain TEXT PRIMARY KEY NOT NULL, + domain_version INTEGER NOT NULL, + revision INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + content_hash TEXT NOT NULL, + legacy_revision INTEGER + ); + CREATE TABLE IF NOT EXISTS ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} ( + domain TEXT NOT NULL, + part_key TEXT NOT NULL, + ordinal INTEGER NOT NULL, + payload TEXT NOT NULL, + content_hash TEXT NOT NULL, + PRIMARY KEY (domain, part_key) + );` +} diff --git a/src/main/persistence/profile-state/profile-state-database-snapshot.ts b/src/main/persistence/profile-state/profile-state-database-snapshot.ts index fd216434819..3f416d4a7f2 100644 --- a/src/main/persistence/profile-state/profile-state-database-snapshot.ts +++ b/src/main/persistence/profile-state/profile-state-database-snapshot.ts @@ -4,6 +4,7 @@ import { basename, dirname, resolve } from 'node:path' import Database from '../../sqlite/sync-database' import { hardenSqliteDatabaseFiles } from '../../sqlite/harden-database-files' import { durableWriteTempPath, renameDurable } from '../../filesystem/durable-file-write' +import { refreshStaleProfileStateLegacyDocuments } from './profile-state-legacy-document-refresh' /** Caller owns the destination and keeps the source connection open until completion. */ export async function writeProfileStateDatabaseSnapshotAsync( @@ -38,6 +39,9 @@ export async function writeProfileStateDatabaseSnapshotAsync( if (snapshot.pragma('journal_mode = DELETE', { simple: true }) !== 'delete') { throw new Error('Profile state snapshot could not become a self-contained database') } + // A restored snapshot outlives any later refresh of the live copy, so older builds + // reading it must already see current legacy rows. + refreshStaleProfileStateLegacyDocuments(snapshot) } finally { snapshot.close() } diff --git a/src/main/persistence/profile-state/profile-state-database.ts b/src/main/persistence/profile-state/profile-state-database.ts index 83fcd713e1c..e641cc463c1 100644 --- a/src/main/persistence/profile-state/profile-state-database.ts +++ b/src/main/persistence/profile-state/profile-state-database.ts @@ -1,4 +1,5 @@ import { withProfileStateWriteTransaction } from './profile-state-write-transaction' +import { ensureProfileStateDocumentPartsTables } from './profile-state-document-parts' import { hasProfileStateAuthorityMarker } from './profile-state-authority-marker' import Database, { isSqliteAvailable } from '../../sqlite/sync-database' import { migrateAutomationRunsStorage } from './profile-state-automation-runs-migration' @@ -113,6 +114,8 @@ export function openProfileStateDatabase( // journal_mode can change the header even without row writes; validate everything first. migrateProfileStateSchema(probe, storedVersion, profileId) configureProfileStatePragmas(probe) + // Additive tables older v3 builds ignore (no schema bump); after busy_timeout is set. + ensureProfileStateDocumentPartsTables(probe) hardenSqliteDatabaseFiles(dbPath) transferred = true return { db: probe, readOnly: false, profileId } diff --git a/src/main/persistence/profile-state/profile-state-document-part-ordinals.ts b/src/main/persistence/profile-state/profile-state-document-part-ordinals.ts new file mode 100644 index 00000000000..cdc9beadaf8 --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-document-part-ordinals.ts @@ -0,0 +1,73 @@ +const ORDINAL_GAP = 1024 + +/** + * Order-preserving ordinals that reuse as many existing values as possible, so inserting, + * removing or moving one key rewrites only that key's row instead of renumbering the rest. + * Keeps the longest run of keys whose previous ordinals are already increasing and spreads + * the others into the gaps between them. + */ +export function assignProfileStatePartOrdinals( + keys: readonly string[], + previous: ReadonlyMap +): number[] { + const kept = keepLongestIncreasing(keys.map((key) => previous.get(key))) + const ordinals: number[] = Array.from({ length: keys.length }, () => 0) + let index = 0 + while (index < keys.length) { + if (kept[index] !== undefined) { + ordinals[index] = kept[index]! + index++ + continue + } + let end = index + while (end < keys.length && kept[end] === undefined) { + end++ + } + const count = end - index + const low = index > 0 ? ordinals[index - 1]! : undefined + const high = end < keys.length ? kept[end]! : undefined + const start = low ?? (high !== undefined ? high - (count + 1) * ORDINAL_GAP : 0) + const stop = high ?? start + (count + 1) * ORDINAL_GAP + const step = Math.floor((stop - start) / (count + 1)) + if (step < 1) { + return keys.map((_, position) => position * ORDINAL_GAP) + } + for (let offset = 0; offset < count; offset++) { + ordinals[index + offset] = start + step * (offset + 1) + } + index = end + } + return ordinals +} + +/** Longest strictly increasing subsequence; other positions become undefined. */ +function keepLongestIncreasing(values: readonly (number | undefined)[]): (number | undefined)[] { + const tails: number[] = [] + const tailIndex: number[] = [] + const parent: number[] = Array.from({ length: values.length }, () => -1) + values.forEach((value, index) => { + if (value === undefined) { + return + } + let low = 0 + let high = tails.length + while (low < high) { + const middle = (low + high) >> 1 + if (tails[middle]! < value) { + low = middle + 1 + } else { + high = middle + } + } + tails[low] = value + tailIndex[low] = index + parent[index] = low > 0 ? tailIndex[low - 1]! : -1 + }) + const kept: (number | undefined)[] = Array.from({ length: values.length }, () => undefined) + let cursor = tailIndex.at(-1) ?? -1 + while (cursor !== -1) { + kept[cursor] = values[cursor] + cursor = parent[cursor]! + } + return kept +} diff --git a/src/main/persistence/profile-state/profile-state-document-parts-reader.ts b/src/main/persistence/profile-state/profile-state-document-parts-reader.ts new file mode 100644 index 00000000000..735d69dd9b3 --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-document-parts-reader.ts @@ -0,0 +1,76 @@ +import type Database from '../../sqlite/sync-database' +import { + isProfileStateSplitDomain, + PROFILE_STATE_SPLIT_DOMAIN_DEPTHS +} from './profile-state-document-parts' +import { readProfileStateSplitDocument, safeInteger } from './profile-state-document-parts-storage' +import { + isRecord, + ProfileStateDocumentCorruptionError, + validateProfileStateDocumentRow, + type ProfileStateDocument +} from './profile-state-document-validation' + +const SPLIT_DOMAIN_LIST = [...PROFILE_STATE_SPLIT_DOMAIN_DEPTHS.keys()] + .map((domain) => `'${domain}'`) + .join(',') + +/** + * Document row columns that skip the legacy payload of split domains: their parts are + * usually authoritative, and that payload can be megabytes. + */ +export const PROFILE_STATE_DOCUMENT_ROW_COLUMNS = `domain, + CASE WHEN domain IN (${SPLIT_DOMAIN_LIST}) THEN NULL ELSE payload END AS payload, + domain_version, revision, updated_at, content_hash` + +/** Validated document for a row selected with PROFILE_STATE_DOCUMENT_ROW_COLUMNS. */ +export function readProfileStateDocumentRow( + db: Database.Database, + row: unknown, + retainParsedValue: boolean +): ProfileStateDocument & { value?: unknown } { + return ( + readProfileStateSplitDocumentForRow(db, row, retainParsedValue) ?? + validateProfileStateDocumentRow(withLegacyPayload(db, row), { retainParsedValue }) + ) +} + +// Runs inside the caller's open statement, so it reads the same snapshot. +function withLegacyPayload(db: Database.Database, row: unknown): unknown { + if (!isRecord(row) || row.payload !== null || typeof row.domain !== 'string') { + return row + } + const legacy: unknown = db + .prepare('SELECT payload FROM profile_state_documents WHERE domain = ?') + .get(row.domain) + return { ...row, payload: isRecord(legacy) ? legacy.payload : null } +} + +/** The reassembled document when this legacy row's domain is stored as parts. */ +export function readProfileStateSplitDocumentForRow( + db: Database.Database, + row: unknown, + retainParsedValue: boolean +): (ProfileStateDocument & { value?: unknown }) | undefined { + const revision = isRecord(row) ? safeInteger(row.revision) : undefined + if ( + !isRecord(row) || + typeof row.domain !== 'string' || + !isProfileStateSplitDomain(row.domain) || + revision === undefined + ) { + return undefined + } + const document = readProfileStateSplitDocument(db, row.domain, revision) + if (!document || !retainParsedValue) { + return document + } + try { + return { ...document, value: JSON.parse(document.payload) } + } catch { + throw new ProfileStateDocumentCorruptionError( + `Profile state document parts are invalid JSON: ${row.domain}`, + row.domain + ) + } +} diff --git a/src/main/persistence/profile-state/profile-state-document-parts-storage.ts b/src/main/persistence/profile-state/profile-state-document-parts-storage.ts new file mode 100644 index 00000000000..9641f09e0fc --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-document-parts-storage.ts @@ -0,0 +1,303 @@ +import type Database from '../../sqlite/sync-database' +import { + assembleProfileStateDomainPayload, + hasProfileStateDocumentPartsTables, + hashProfileStatePart, + isProfileStateSplitDomain, + PROFILE_STATE_DOCUMENT_PARTS_META_TABLE, + PROFILE_STATE_DOCUMENT_PARTS_TABLE, + PROFILE_STATE_SPLIT_DOMAIN_DEPTHS, + splitProfileStateDomainValue, + type ProfileStateDocumentPart +} from './profile-state-document-parts' +import { + hashProfileStatePayload, + isRecord, + ProfileStateDocumentCorruptionError, + type ProfileStateDocument +} from './profile-state-document-validation' +import { assignProfileStatePartOrdinals } from './profile-state-document-part-ordinals' + +type PartsMeta = { + domainVersion: number + revision: number + updatedAt: number + contentHash: string + legacyRevision: number | null +} + +function readPartsMeta(db: Database.Database, domain: string): PartsMeta | undefined { + if (!hasProfileStateDocumentPartsTables(db)) { + return undefined + } + const row = db + .prepare( + `SELECT domain_version, revision, updated_at, content_hash, legacy_revision + FROM ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE} WHERE domain = ?` + ) + .get(domain) + if (row === undefined) { + return undefined + } + const domainVersion = isRecord(row) ? safeInteger(row.domain_version) : undefined + const revision = isRecord(row) ? safeInteger(row.revision) : undefined + const updatedAt = isRecord(row) ? safeInteger(row.updated_at) : undefined + const legacyRevision = isRecord(row) ? safeInteger(row.legacy_revision) : undefined + if ( + !isRecord(row) || + // Same bounds as an ordinary document row, which this metadata stands in for. + domainVersion === undefined || + domainVersion < 1 || + revision === undefined || + revision < 1 || + updatedAt === undefined || + updatedAt < 0 || + typeof row.content_hash !== 'string' || + !/^[a-f0-9]{64}$/.test(row.content_hash) || + (row.legacy_revision !== null && (legacyRevision === undefined || legacyRevision < 1)) + ) { + throw new ProfileStateDocumentCorruptionError( + `Profile state document parts metadata is invalid: ${domain}`, + domain + ) + } + return { + domainVersion, + revision, + updatedAt, + contentHash: row.content_hash, + legacyRevision: legacyRevision ?? null + } +} + +export function safeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isSafeInteger(value) ? value : undefined +} + +export function readLegacyRevision(db: Database.Database, domain: string): number | null { + const row = db + .prepare('SELECT revision FROM profile_state_documents WHERE domain = ?') + .get(domain) + return (isRecord(row) ? safeInteger(row.revision) : undefined) ?? null +} + +/** Parts win only while the legacy row is the one this layout last synced. */ +export function authoritativeMeta( + db: Database.Database, + domain: string, + legacyRevision: number | null +): PartsMeta | undefined { + const meta = readPartsMeta(db, domain) + return meta !== undefined && legacyRevision !== null && meta.legacyRevision === legacyRevision + ? meta + : undefined +} + +/** Reassembled document for a split domain, or undefined when its legacy row is authoritative. */ +export function readProfileStateSplitDocument( + db: Database.Database, + domain: string, + legacyRevision: number +): ProfileStateDocument | undefined { + if (!isProfileStateSplitDomain(domain)) { + return undefined + } + const meta = authoritativeMeta(db, domain, legacyRevision) + if (!meta) { + return undefined + } + const rows = db + .prepare( + `SELECT part_key, payload, content_hash FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} + WHERE domain = ? ORDER BY ordinal` + ) + .iterate(domain) + const parts: ProfileStateDocumentPart[] = [] + for (const row of rows) { + if ( + !isRecord(row) || + typeof row.part_key !== 'string' || + typeof row.payload !== 'string' || + row.content_hash !== hashProfileStatePart(row.payload) + ) { + throw new ProfileStateDocumentCorruptionError( + `Profile state document part is invalid: ${domain}`, + domain + ) + } + parts.push({ key: row.part_key, payload: row.payload }) + } + let payload: string + try { + payload = assembleProfileStateDomainPayload(parts) + } catch { + throw new ProfileStateDocumentCorruptionError( + `Profile state document parts are invalid: ${domain}`, + domain + ) + } + if (hashProfileStatePayload(payload) !== meta.contentHash) { + throw new ProfileStateDocumentCorruptionError( + `Profile state document parts hash mismatch: ${domain}`, + domain + ) + } + return { + domain, + payload, + domainVersion: meta.domainVersion, + revision: meta.revision, + updatedAt: meta.updatedAt, + contentHash: meta.contentHash + } +} + +export type SplitDomainWriteState = + | { kind: 'unsplit' } + | { kind: 'parts'; legacyRevision: number; contentHash: string; revision: number } + +/** Lets the writer compare by hash without reading or re-validating a stale legacy payload. */ +export function readSplitDomainWriteState( + db: Database.Database, + domain: string +): SplitDomainWriteState { + if (!isProfileStateSplitDomain(domain)) { + return { kind: 'unsplit' } + } + const legacyRevision = readLegacyRevision(db, domain) + const meta = authoritativeMeta(db, domain, legacyRevision) + return meta && legacyRevision !== null + ? { kind: 'parts', legacyRevision, contentHash: meta.contentHash, revision: meta.revision } + : { kind: 'unsplit' } +} + +export function deleteProfileStateDomainParts(db: Database.Database, domain: string): void { + if (!hasProfileStateDocumentPartsTables(db)) { + return + } + db.prepare(`DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} WHERE domain = ?`).run(domain) + db.prepare(`DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE} WHERE domain = ?`).run(domain) +} + +export function clearProfileStateDocumentParts(db: Database.Database): void { + if (!hasProfileStateDocumentPartsTables(db)) { + return + } + db.exec(`DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE}; + DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE};`) +} + +/** + * Write only the parts whose content or position changed. Returns false when the payload + * cannot be split losslessly; the caller then stores it as an ordinary legacy row. + */ +export function writeProfileStateDomainParts( + db: Database.Database, + args: { + domain: string + payload: string + /** Already-parsed payload, when the caller has it. */ + value?: unknown + payloadHash: string + domainVersion: number + revision: number + updatedAt: number + /** Legacy row revision after this write; parts stay authoritative only while it matches. */ + legacyRevision: number + /** Existing parts are a valid base to diff against. */ + diffExisting: boolean + } +): boolean { + if (!hasProfileStateDocumentPartsTables(db)) { + return false + } + const depth = PROFILE_STATE_SPLIT_DOMAIN_DEPTHS.get(args.domain) + if (depth === undefined) { + return false + } + const parts = splitProfileStateDomainValue( + args.value === undefined ? JSON.parse(args.payload) : args.value, + depth + ) + // Non-canonical JSON (whitespace, escapes) would not round-trip byte-for-byte. + if (assembleProfileStateDomainPayload(parts) !== args.payload) { + deleteProfileStateDomainParts(db, args.domain) + return false + } + const existing = new Map() + if (args.diffExisting) { + for (const row of db + .prepare( + `SELECT part_key, ordinal, content_hash FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} + WHERE domain = ?` + ) + .iterate(args.domain)) { + if (isRecord(row) && typeof row.part_key === 'string') { + existing.set(row.part_key, { + ordinal: Number(row.ordinal), + contentHash: String(row.content_hash) + }) + } + } + } else { + db.prepare(`DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} WHERE domain = ?`).run( + args.domain + ) + } + const ordinals = assignProfileStatePartOrdinals( + parts.map(({ key }) => key), + new Map([...existing].map(([key, { ordinal }]) => [key, ordinal])) + ) + const insert = db.prepare( + `INSERT INTO ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} + (domain, part_key, ordinal, payload, content_hash) VALUES (?, ?, ?, ?, ?)` + ) + const updatePayload = db.prepare( + `UPDATE ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} SET ordinal = ?, payload = ?, content_hash = ? + WHERE domain = ? AND part_key = ?` + ) + const updateOrdinal = db.prepare( + `UPDATE ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} SET ordinal = ? WHERE domain = ? AND part_key = ?` + ) + const remove = db.prepare( + `DELETE FROM ${PROFILE_STATE_DOCUMENT_PARTS_TABLE} WHERE domain = ? AND part_key = ?` + ) + const seen = new Set() + parts.forEach((part, index) => { + seen.add(part.key) + const ordinal = ordinals[index]! + const previous = existing.get(part.key) + const contentHash = hashProfileStatePart(part.payload) + if (!previous) { + insert.run(args.domain, part.key, ordinal, part.payload, contentHash) + } else if (previous.contentHash !== contentHash) { + updatePayload.run(ordinal, part.payload, contentHash, args.domain, part.key) + } else if (previous.ordinal !== ordinal) { + updateOrdinal.run(ordinal, args.domain, part.key) + } + }) + for (const key of existing.keys()) { + if (!seen.has(key)) { + remove.run(args.domain, key) + } + } + db.prepare( + `INSERT INTO ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE} + (domain, domain_version, revision, updated_at, content_hash, legacy_revision) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(domain) DO UPDATE SET + domain_version = excluded.domain_version, + revision = excluded.revision, + updated_at = excluded.updated_at, + content_hash = excluded.content_hash, + legacy_revision = excluded.legacy_revision` + ).run( + args.domain, + args.domainVersion, + args.revision, + args.updatedAt, + args.payloadHash, + args.legacyRevision + ) + return true +} diff --git a/src/main/persistence/profile-state/profile-state-document-parts.test.ts b/src/main/persistence/profile-state/profile-state-document-parts.test.ts new file mode 100644 index 00000000000..608b1f7f764 --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-document-parts.test.ts @@ -0,0 +1,168 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { exportProfileStateJson, readProfileStateRevision } from './profile-state-documents' +import { + openProfileStateDatabase, + openProfileStateDatabaseReadOnly, + profileStateDatabaseFile +} from './profile-state-database' +import { writeProfileStateDomains } from './profile-state-domain-writes' +import { readProfileStateDomainsWithRevisionFromDatabase } from './profile-state-domain-reader' +import { hashProfileStatePayload, isRecord } from './profile-state-document-validation' +import { writeProfileStateBackup } from './profile-state-backup-job' +import { ProfileStateSqliteAuthority } from './profile-state-sqlite-authority' + +const directories: string[] = [] + +afterEach(() => { + vi.useRealTimers() + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function databasePath(): string { + const directory = mkdtempSync(join(tmpdir(), 'orca-profile-state-parts-')) + directories.push(directory) + return profileStateDatabaseFile(directory) +} + +function open() { + return openProfileStateDatabase(databasePath(), 'profile-a').db +} + +type Db = ReturnType + +function write(db: Db, domains: Record): void { + writeProfileStateDomains(db, { + expectedRevision: readProfileStateRevision(db), + replacements: Object.entries(domains).map(([domain, payload]) => ({ domain, payload })) + }) +} + +function read(db: Db, domain: string): unknown { + const result = readProfileStateDomainsWithRevisionFromDatabase(db, [domain]) + if (result.kind !== 'values') { + throw result.error + } + return result.values.get(domain) +} + +/** What a build without per-entry storage reads: only profile_state_documents. */ +function readAsOlderBuild(path: string, domain: string): string { + const { db } = openProfileStateDatabaseReadOnly(path, 'profile-a') + try { + const row: unknown = db + .prepare('SELECT payload FROM profile_state_documents WHERE domain = ?') + .get(domain) + return isRecord(row) && typeof row.payload === 'string' ? row.payload : '' + } finally { + db.close() + } +} + +const v1 = + '{"runtime:a":{"tabsByWorktree":{"wt1":[{"id":"t1","title":"shell"}],"wt2":[{"id":"t2"}]},"activeTabId":"t1"},"ssh:b":{"tabsByWorktree":{}}}' +const v2 = + '{"runtime:a":{"tabsByWorktree":{"wt2":[{"id":"t2"}],"wt3":[],"wt1":[{"id":"t1","title":"renamed"}]},"activeTabId":"t1"},"ssh:b":{"tabsByWorktree":{}}}' + +describe('profile state document parts', () => { + it('reads back the exact document after entries change, appear and move', () => { + const db = open() + try { + write(db, { workspaceSessionsByHostId: v1, other: '1' }) + write(db, { workspaceSessionsByHostId: v2 }) + + expect(exportProfileStateJson(db)).toBe(`{"workspaceSessionsByHostId":${v2},"other":1}`) + } finally { + db.close() + } + }) + + it('gives an older build current state after a crash once the writer reopens', () => { + const path = databasePath() + openProfileStateDatabase(path, 'profile-a').db.close() + const crashed = new ProfileStateSqliteAuthority(path, 'profile-a') + crashed.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v1 }]) + crashed.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v2 }]) + // A crash never runs the graceful-close refresh. + crashed.close() + + const reopened = new ProfileStateSqliteAuthority(path, 'profile-a') + try { + reopened.writeSerializedDomains([{ domain: 'other', payload: '1' }]) + expect(readAsOlderBuild(path, 'workspaceSessionsByHostId')).toBe(v2) + } finally { + reopened.close() + } + }) + + it('trusts a document an older build rewrote, then keeps later writes', () => { + const db = open() + try { + write(db, { workspaceSessionsByHostId: v1 }) + write(db, { workspaceSessionsByHostId: v2 }) + // An older build knows only profile_state_documents. + const olderPayload = '{"runtime:a":{"activeTabId":"from-older-build"}}' + const revision = readProfileStateRevision(db) + 1 + db.prepare( + `UPDATE profile_state_documents SET payload = ?, content_hash = ?, revision = ? + WHERE domain = 'workspaceSessionsByHostId'` + ).run(olderPayload, hashProfileStatePayload(olderPayload), revision) + db.prepare(`UPDATE profile_state_meta SET value = ? WHERE key = 'revision'`).run( + String(revision) + ) + + expect(read(db, 'workspaceSessionsByHostId')).toEqual({ + 'runtime:a': { activeTabId: 'from-older-build' } + }) + + write(db, { workspaceSessionsByHostId: v1 }) + write(db, { workspaceSessionsByHostId: v2 }) + expect(exportProfileStateJson(db)).toBe(`{"workspaceSessionsByHostId":${v2}}`) + } finally { + db.close() + } + }) + + it('gives an older build current state after writes stop, without another write', () => { + const path = databasePath() + openProfileStateDatabase(path, 'profile-a').db.close() + const first = new ProfileStateSqliteAuthority(path, 'profile-a') + first.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v1 }]) + const revision = first.revision + first.close() + + vi.useFakeTimers() + const writer = new ProfileStateSqliteAuthority(path, 'profile-a') + try { + writer.initializeFromRevision(revision) + writer.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v2 }]) + vi.advanceTimersByTime(60 * 60_000) + expect(readAsOlderBuild(path, 'workspaceSessionsByHostId')).toBe(v2) + } finally { + writer.close() + } + }) + + it('publishes backups whose older-build copy is current', async () => { + const path = databasePath() + openProfileStateDatabase(path, 'profile-a').db.close() + const writer = new ProfileStateSqliteAuthority(path, 'profile-a') + try { + writer.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v1 }]) + writer.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v2 }]) + const backup = join(directories.at(-1)!, 'backup.db') + await writeProfileStateBackup({ + databasePath: path, + profileId: 'profile-a', + targetPath: backup + }) + expect(readAsOlderBuild(backup, 'workspaceSessionsByHostId')).toBe(v2) + } finally { + writer.close() + } + }) +}) diff --git a/src/main/persistence/profile-state/profile-state-document-parts.ts b/src/main/persistence/profile-state/profile-state-document-parts.ts new file mode 100644 index 00000000000..ac100651fa8 --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-document-parts.ts @@ -0,0 +1,150 @@ +import type Database from '../../sqlite/sync-database' +import { hashProfileStatePayload, isRecord } from './profile-state-document-validation' +import { WORKSPACE_SESSION_DOMAINS } from '../workspace-session-domains' + +/** + * Large map-shaped domains are stored as one row per entry so a save rewrites only the + * entries that changed. Readers reassemble the exact original JSON, so everything above + * this layer still sees one document per domain. + * + * Older builds ignore these tables and read the domain's legacy `profile_state_documents` + * row. That row is refreshed from the parts when the writer starts, at most every five minutes while it runs, and at graceful close; when an + * older build rewrites it, its revision no longer matches `legacy_revision` and the legacy + * row becomes authoritative again until the next write re-splits it. + */ +export { + PROFILE_STATE_DOCUMENT_PARTS_META_TABLE, + PROFILE_STATE_DOCUMENT_PARTS_TABLE +} from './profile-state-database-schema' +import { + createProfileStateDocumentPartsTablesSql, + PROFILE_STATE_DOCUMENT_PARTS_META_TABLE, + PROFILE_STATE_DOCUMENT_PARTS_TABLE +} from './profile-state-database-schema' + +const [LOCAL_SESSION_DOMAIN, HOST_SESSION_DOMAIN] = WORKSPACE_SESSION_DOMAINS + +/** Object nesting levels split into rows; arrays and scalars are always leaves. */ +export const PROFILE_STATE_SPLIT_DOMAIN_DEPTHS: ReadonlyMap = new Map([ + // host → session field → tab/worktree/pane entry + [HOST_SESSION_DOMAIN, 3], + // session field → tab/worktree/pane entry + [LOCAL_SESSION_DOMAIN, 2], + // environment → { workspaces } → worktree + ['remoteLayoutSnapshots', 3], + ['worktreeMeta', 1], + ['worktreeMetaByIdentity', 1], + ['worktreeIdentityAliases', 1] +]) + +export function isProfileStateSplitDomain(domain: string): boolean { + return PROFILE_STATE_SPLIT_DOMAIN_DEPTHS.has(domain) +} + +export type ProfileStateDocumentPart = { + /** JSON array of the object keys leading to this value. */ + key: string + payload: string +} + +/** Depth-first parts whose concatenation reproduces JSON.stringify(value) exactly. */ +export function splitProfileStateDomainValue( + value: unknown, + depth: number +): ProfileStateDocumentPart[] { + const parts: ProfileStateDocumentPart[] = [] + const visit = (current: unknown, path: string[], remaining: number): void => { + if (remaining > 0 && isRecord(current) && typeof current.toJSON !== 'function') { + // JSON.stringify drops undefined/function/symbol members; the parts must match it. + const children = Object.entries(current).filter( + ([, child]) => + child !== undefined && typeof child !== 'function' && typeof child !== 'symbol' + ) + if (children.length > 0) { + for (const [key, child] of children) { + visit(child, [...path, key], remaining - 1) + } + return + } + } + const payload = JSON.stringify(current) + if (payload !== undefined) { + parts.push({ key: JSON.stringify(path), payload }) + } + } + visit(value, [], depth) + return parts +} + +/** Inverse of split: rebuild the document text from depth-first ordered parts. */ +export function assembleProfileStateDomainPayload( + parts: readonly ProfileStateDocumentPart[] +): string { + let out = '' + let open: string[] = [] + let first = true + for (const part of parts) { + const path = parsePartKey(part.key) + let common = 0 + while (common < open.length && common < path.length - 1 && open[common] === path[common]) { + common++ + } + for (let i = open.length; i > common; i--) { + out += '}' + } + open = open.slice(0, common) + if (!first) { + out += ',' + } + for (let i = common; i < path.length - 1; i++) { + out += `${JSON.stringify(path[i])}:{` + open.push(path[i]!) + } + if (path.length > 0) { + out += `${JSON.stringify(path.at(-1))}:` + } + out += part.payload + first = false + } + out += '}'.repeat(open.length) + if (parts.length === 0) { + return '{}' + } + return parts.length === 1 && parsePartKey(parts[0]!.key).length === 0 ? out : `{${out}}` +} + +function parsePartKey(key: string): string[] { + const path: unknown = JSON.parse(key) + if (!Array.isArray(path) || !path.every((segment) => typeof segment === 'string')) { + throw new Error('Profile state document part key is invalid') + } + return path +} + +export function hashProfileStatePart(payload: string): string { + return hashProfileStatePayload(payload) +} + +const partsTablesByDatabase = new WeakMap() + +/** Read-only and pre-split databases may not have the tables; treat them as unsplit. */ +export function hasProfileStateDocumentPartsTables(db: Database.Database): boolean { + const cached = partsTablesByDatabase.get(db) + if (cached !== undefined) { + return cached + } + const rows = db + .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)`) + .all(PROFILE_STATE_DOCUMENT_PARTS_TABLE, PROFILE_STATE_DOCUMENT_PARTS_META_TABLE) + const present = rows.length === 2 + // Only cache presence: a writable open may create the tables after a negative probe. + if (present) { + partsTablesByDatabase.set(db, true) + } + return present +} + +export function ensureProfileStateDocumentPartsTables(db: Database.Database): void { + db.exec(createProfileStateDocumentPartsTablesSql()) + partsTablesByDatabase.set(db, true) +} diff --git a/src/main/persistence/profile-state/profile-state-document-reader.ts b/src/main/persistence/profile-state/profile-state-document-reader.ts index 221a89b3176..39580c19b56 100644 --- a/src/main/persistence/profile-state/profile-state-document-reader.ts +++ b/src/main/persistence/profile-state/profile-state-document-reader.ts @@ -4,9 +4,12 @@ import { assertProfileStateDocumentRevision, readProfileStateRevision } from './profile-state-revision' +import { + PROFILE_STATE_DOCUMENT_ROW_COLUMNS, + readProfileStateDocumentRow +} from './profile-state-document-parts-reader' import { ProfileStateDocumentCorruptionError, - validateProfileStateDocumentRow, type ProfileStateDocument, type ProfileStateParsedDocument, type ProfileStateValidatedDocument @@ -45,7 +48,7 @@ export function readProfileStateDocuments( : readProfileStateAutomationRunsDocument(db, profileRevision) const rows = db .prepare( - `SELECT domain, payload, domain_version, revision, updated_at, content_hash + `SELECT ${PROFILE_STATE_DOCUMENT_ROW_COLUMNS} FROM profile_state_documents ORDER BY rowid` ) .iterate() @@ -55,9 +58,7 @@ export function readProfileStateDocuments( | ProfileStateValidatedDocument )[] = [] for (const row of rows) { - const document = validateProfileStateDocumentRow(row, { - retainParsedValue: options.representation === 'parsed' - }) + const document = readProfileStateDocumentRow(db, row, options.representation === 'parsed') assertProfileStateDocumentRevision(document.revision, profileRevision, document.domain) if ( normalized !== undefined && diff --git a/src/main/persistence/profile-state/profile-state-documents.ts b/src/main/persistence/profile-state/profile-state-documents.ts index 79ab282cc28..ca0fa1fdc70 100644 --- a/src/main/persistence/profile-state/profile-state-documents.ts +++ b/src/main/persistence/profile-state/profile-state-documents.ts @@ -19,6 +19,7 @@ import { } from './profile-state-automation-runs' import { readProfileStateDocuments } from './profile-state-document-reader' +import { clearProfileStateDocumentParts } from './profile-state-document-parts-storage' import { readProfileStateRevision } from './profile-state-revision' import { readProfileStateJsonAcceptance } from './legacy-json/profile-state-json-acceptance' @@ -95,6 +96,8 @@ export function importProfileStateJson( } const revision = actualRevision + 1 clearProfileStateAutomationRuns(db) + // Imported rows are authoritative; the next write of a split domain re-splits it. + clearProfileStateDocumentParts(db) db.exec('DELETE FROM profile_state_documents') const automationRuns = entries.find((entry) => entry.domain === 'automationRuns') const normalizedRuns = diff --git a/src/main/persistence/profile-state/profile-state-domain-reader.ts b/src/main/persistence/profile-state/profile-state-domain-reader.ts index 8e87a6d5bfa..103813c213d 100644 --- a/src/main/persistence/profile-state/profile-state-domain-reader.ts +++ b/src/main/persistence/profile-state/profile-state-domain-reader.ts @@ -6,9 +6,10 @@ import { import { readProfileStateAutomationRunsDocument } from './profile-state-automation-runs' import { openProfileStateDatabaseReadOnly } from './profile-state-database' import { - validateProfileStateDocumentRow, - type ProfileStateParsedDocument -} from './profile-state-document-validation' + PROFILE_STATE_DOCUMENT_ROW_COLUMNS, + readProfileStateDocumentRow +} from './profile-state-document-parts-reader' +import type { ProfileStateParsedDocument } from './profile-state-document-validation' export type ProfileStateDomainReadResult = | { kind: 'value'; value: unknown } @@ -96,16 +97,14 @@ function readSelectedDocuments( const placeholders = domains.map(() => '?').join(',') const rows = db .prepare( - `SELECT domain, payload, domain_version, revision, updated_at, content_hash + `SELECT ${PROFILE_STATE_DOCUMENT_ROW_COLUMNS} FROM profile_state_documents WHERE domain IN (${placeholders}) ORDER BY rowid` ) .iterate(...domains) return Array.from(rows, (row) => { - const { payload: _payload, ...document } = validateProfileStateDocumentRow(row, { - retainParsedValue: true - }) + const { payload: _payload, ...document } = readProfileStateDocumentRow(db, row, true) return { ...document, value: document.value } }) } diff --git a/src/main/persistence/profile-state/profile-state-domain-write-validation.ts b/src/main/persistence/profile-state/profile-state-domain-write-validation.ts index e2216e14018..b899e477496 100644 --- a/src/main/persistence/profile-state/profile-state-domain-write-validation.ts +++ b/src/main/persistence/profile-state/profile-state-domain-write-validation.ts @@ -1,4 +1,5 @@ import { PROFILE_STATE_DOCUMENT_VERSION } from './profile-state-database-schema' +import { isProfileStateSplitDomain } from './profile-state-document-parts' import { hashProfileStateJson } from './profile-state-documents' import type { AutomationRunsWritePreparation } from './profile-state-automation-runs' import type { @@ -11,6 +12,8 @@ export type PreparedProfileStateMutation = ProfileStateDomainMutation & { payloadHash: string | null // Retain the checked history only for this write to avoid reparsing during normalization. automationRunsValue?: unknown + /** Parsed payload of a split domain, so storing its parts does not parse it again. */ + splitDomainValue?: unknown automationRuns?: AutomationRunsWritePreparation } @@ -58,7 +61,8 @@ export function prepareProfileStateDomainMutation( now: replacement.now, domainVersion: replacement.domainVersion ?? PROFILE_STATE_DOCUMENT_VERSION, payloadHash, - automationRunsValue: replacement.domain === 'automationRuns' ? parsed : undefined + automationRunsValue: replacement.domain === 'automationRuns' ? parsed : undefined, + splitDomainValue: isProfileStateSplitDomain(replacement.domain) ? parsed : undefined } } diff --git a/src/main/persistence/profile-state/profile-state-domain-writes.ts b/src/main/persistence/profile-state/profile-state-domain-writes.ts index bf0205f3681..44314d9ab3b 100644 --- a/src/main/persistence/profile-state/profile-state-domain-writes.ts +++ b/src/main/persistence/profile-state/profile-state-domain-writes.ts @@ -17,6 +17,12 @@ import { } from './profile-state-automation-runs' import { isRecord, validateProfileStateDocumentRow } from './profile-state-document-validation' import { assertProfileStateDocumentRevision } from './profile-state-revision' +import { + deleteProfileStateDomainParts, + readSplitDomainWriteState, + writeProfileStateDomainParts +} from './profile-state-document-parts-storage' +import { isProfileStateSplitDomain } from './profile-state-document-parts' import { prepareProfileStateDomainMutation, validateProfileStateDomainTransaction, @@ -90,7 +96,18 @@ export function writeProfileStateDomains( const currentAutomationRuns = readCurrentAutomationRunsState(db, actualRevision) const updates: PreparedProfileStateMutation[] = [] + const splitStates = new Map>() for (const mutation of prepared) { + const splitState = readSplitDomainWriteState(db, mutation.domain) + splitStates.set(mutation.domain, splitState) + if (splitState.kind === 'parts') { + // Compare by hash; the legacy row is a stale downgrade copy and is not re-read. + assertProfileStateDocumentRevision(splitState.revision, actualRevision, mutation.domain) + if (mutation.payload === null || mutation.payloadHash !== splitState.contentHash) { + updates.push(mutation) + } + continue + } const existing = db .prepare( `SELECT domain, payload, domain_version, revision, updated_at, content_hash @@ -172,11 +189,29 @@ export function writeProfileStateDomains( } if (mutation.payload === null) { db.prepare('DELETE FROM profile_state_documents WHERE domain = ?').run(mutation.domain) + deleteProfileStateDomainParts(db, mutation.domain) } else { const updatedAt = (mutation.now ?? Date.now)() if (!Number.isSafeInteger(updatedAt) || updatedAt < 0) { throw new Error(`Profile state domain update timestamp is invalid: ${mutation.domain}`) } + const splitState = splitStates.get(mutation.domain) + if ( + splitState?.kind === 'parts' && + writeProfileStateDomainParts(db, { + domain: mutation.domain, + payload: mutation.payload, + value: mutation.splitDomainValue, + payloadHash: mutation.payloadHash!, + domainVersion: mutation.domainVersion, + revision: nextRevision, + updatedAt, + legacyRevision: splitState.legacyRevision, + diffExisting: true + }) + ) { + continue + } db.prepare( `INSERT INTO profile_state_documents (domain, payload, domain_version, revision, updated_at, content_hash) @@ -195,6 +230,20 @@ export function writeProfileStateDomains( updatedAt, mutation.payloadHash ) + if (isProfileStateSplitDomain(mutation.domain)) { + // The legacy row now matches; later writes touch only the parts that change. + writeProfileStateDomainParts(db, { + domain: mutation.domain, + payload: mutation.payload, + value: mutation.splitDomainValue, + payloadHash: mutation.payloadHash!, + domainVersion: mutation.domainVersion, + revision: nextRevision, + updatedAt, + legacyRevision: nextRevision, + diffExisting: false + }) + } } } db.prepare( diff --git a/src/main/persistence/profile-state/profile-state-legacy-document-refresh.ts b/src/main/persistence/profile-state/profile-state-legacy-document-refresh.ts new file mode 100644 index 00000000000..fb58a751409 --- /dev/null +++ b/src/main/persistence/profile-state/profile-state-legacy-document-refresh.ts @@ -0,0 +1,137 @@ +import type Database from '../../sqlite/sync-database' +import { + hasProfileStateDocumentPartsTables, + PROFILE_STATE_DOCUMENT_PARTS_META_TABLE, + PROFILE_STATE_SPLIT_DOMAIN_DEPTHS +} from './profile-state-document-parts' +import { + authoritativeMeta, + readLegacyRevision, + readProfileStateSplitDocument +} from './profile-state-document-parts-storage' +import { withProfileStateWriteTransaction } from './profile-state-write-transaction' + +/** + * Older builds read the legacy rows. Starting the writer brings them current (covering an + * earlier crash); while it runs they lag the parts by at most this long, even when writes + * stop, which bounds what a downgrade straight after a crash can miss. Each refresh rewrites + * only stale rows. + */ +const LEGACY_DOCUMENT_REFRESH_INTERVAL_MS = 5 * 60 * 1000 + +type DatabaseSource = () => Database.Database | undefined + +export class ProfileStateLegacyDocumentRefresh { + private refreshedAt = Date.now() + private caughtUp = false + private timer: ReturnType | undefined + private scheduled = false + + /** + * The writer thread also refreshes on a timer through quiet periods; other authorities run + * on threads that must not block on a refresh, so they only refresh around their own writes. + */ + startWriterThread(db: Database.Database | undefined): void { + this.scheduled = true + this.catchUp(db) + } + + /** A crash skips the close-time refresh; catch older builds up once before writing on. */ + catchUp(db: Database.Database | undefined): void { + if (!this.caughtUp) { + this.caughtUp = true + this.refresh(db) + } + } + + beforeClose(db: Database.Database | undefined, graceful: boolean | undefined): void { + clearTimeout(this.timer) + this.timer = undefined + if (graceful) { + this.refresh(db) + } + } + + afterWrite(source: DatabaseSource): void { + const remaining = this.refreshedAt + LEGACY_DOCUMENT_REFRESH_INTERVAL_MS - Date.now() + if (remaining <= 0) { + this.refresh(source()) + } else if (this.scheduled && this.timer === undefined) { + this.timer = setTimeout(() => { + this.timer = undefined + this.refresh(source()) + }, remaining) + this.timer.unref?.() + } + } + + /** Best effort: a failed refresh rolls back, leaving older builds a staler copy, never a wrong one. */ + refresh(db: Database.Database | undefined): void { + // An open transaction means a write is unsettled; the next refresh will catch up. + if (!db || db.isTransaction) { + return + } + clearTimeout(this.timer) + this.timer = undefined + this.refreshedAt = Date.now() + try { + refreshStaleProfileStateLegacyDocuments(db) + } catch (error) { + console.warn('[profile-state] Failed to refresh legacy documents:', error) + } + } +} + +/** Throws on failure; the transaction leaves legacy rows as they were. */ +export function refreshStaleProfileStateLegacyDocuments(db: Database.Database): void { + // Most checks find nothing stale; skip the write transaction entirely then. + if (staleLegacyDomains(db).length > 0) { + withProfileStateWriteTransaction(db, () => refreshProfileStateLegacyDocuments(db)) + } +} + +function staleLegacyDomains(db: Database.Database): { domain: string; legacyRevision: number }[] { + if (!hasProfileStateDocumentPartsTables(db)) { + return [] + } + const stale: { domain: string; legacyRevision: number }[] = [] + for (const domain of PROFILE_STATE_SPLIT_DOMAIN_DEPTHS.keys()) { + const legacyRevision = readLegacyRevision(db, domain) + if (legacyRevision === null) { + continue + } + const meta = authoritativeMeta(db, domain, legacyRevision) + if (meta && meta.revision !== legacyRevision) { + stale.push({ domain, legacyRevision }) + } + } + return stale +} + +/** + * Bring stale legacy rows up to date so an older build (or a downgrade) reads current + * state. The profile revision does not move: the logical content is unchanged. + */ +function refreshProfileStateLegacyDocuments(db: Database.Database): void { + for (const { domain, legacyRevision } of staleLegacyDomains(db)) { + const document = readProfileStateSplitDocument(db, domain, legacyRevision) + if (!document) { + continue + } + db.prepare( + `UPDATE profile_state_documents + SET payload = ?, domain_version = ?, revision = ?, updated_at = ?, content_hash = ? + WHERE domain = ?` + ).run( + document.payload, + document.domainVersion, + document.revision, + document.updatedAt, + document.contentHash, + domain + ) + db.prepare( + `UPDATE ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE} SET legacy_revision = ? WHERE domain = ?` + ).run(document.revision, domain) + } +} diff --git a/src/main/persistence/profile-state/profile-state-revision.ts b/src/main/persistence/profile-state/profile-state-revision.ts index ec4dd57454d..44c49af54b5 100644 --- a/src/main/persistence/profile-state/profile-state-revision.ts +++ b/src/main/persistence/profile-state/profile-state-revision.ts @@ -5,6 +5,10 @@ import { PROFILE_STATE_AUTOMATION_RUNS_TABLE } from './profile-state-automation-runs-model' import { isRecord, ProfileStateDocumentCorruptionError } from './profile-state-document-validation' +import { + hasProfileStateDocumentPartsTables, + PROFILE_STATE_DOCUMENT_PARTS_META_TABLE +} from './profile-state-document-parts' export function readProfileStateRevision(db: Database.Database): number { const row = db @@ -27,7 +31,12 @@ export function assertProfileStateRepairRevision(db: Database.Database, revision `SELECT 1 FROM ( SELECT revision FROM profile_state_documents UNION ALL SELECT revision FROM ${PROFILE_STATE_AUTOMATION_RUNS_TABLE} - UNION ALL SELECT revision FROM ${PROFILE_STATE_AUTOMATION_RUNS_META_TABLE} + UNION ALL SELECT revision FROM ${PROFILE_STATE_AUTOMATION_RUNS_META_TABLE}${ + hasProfileStateDocumentPartsTables(db) + ? ` + UNION ALL SELECT revision FROM ${PROFILE_STATE_DOCUMENT_PARTS_META_TABLE}` + : '' + } ) WHERE typeof(revision) IN ('integer', 'real') AND revision BETWEEN 1 AND ? AND revision = CAST(revision AS INTEGER) AND revision > ? LIMIT 1` diff --git a/src/main/persistence/profile-state/profile-state-section-repair.ts b/src/main/persistence/profile-state/profile-state-section-repair.ts index 71669a608fe..344f1c78baa 100644 --- a/src/main/persistence/profile-state/profile-state-section-repair.ts +++ b/src/main/persistence/profile-state/profile-state-section-repair.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from 'node:fs' +import { readProfileStateSplitDocumentForRow } from './profile-state-document-parts-reader' import type Database from '../../sqlite/sync-database' import { ProfileStateRecoveryCommandError, @@ -237,7 +238,9 @@ function scanProfileStateSections(db: Database.Database): SectionScan | undefine const domain = isRecord(row) && typeof row.domain === 'string' ? row.domain : undefined let document: ReturnType try { - document = validateProfileStateDocumentRow(row, { retainParsedValue: true }) + document = + readProfileStateSplitDocumentForRow(db, row, true) ?? + validateProfileStateDocumentRow(row, { retainParsedValue: true }) } catch (error) { if (!(error instanceof ProfileStateDocumentCorruptionError) || domain === undefined) { throw error diff --git a/src/main/persistence/profile-state/profile-state-sqlite-authority.ts b/src/main/persistence/profile-state/profile-state-sqlite-authority.ts index 7aef0e7e026..bd15fa2545a 100644 --- a/src/main/persistence/profile-state/profile-state-sqlite-authority.ts +++ b/src/main/persistence/profile-state/profile-state-sqlite-authority.ts @@ -35,6 +35,7 @@ import { buildCompleteDocumentReplacements } from './profile-state-complete-repl import { ProfileStateBackupRotation } from './profile-state-backup-rotation' import { ensureProfileStateAuthorityMarker } from './profile-state-authority-marker' import type { ProfileStateWriterInitialization } from './profile-state-writer-protocol' +import { ProfileStateLegacyDocumentRefresh } from './profile-state-legacy-document-refresh' /** Reuse one writable handle; fenced domain commits preserve unchanged normalized rows. */ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { @@ -42,6 +43,7 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { private observedRevision: number | undefined private writableDatabase: ReturnType | undefined private backupRotation: ProfileStateBackupRotation | undefined + private readonly legacyDocuments = new ProfileStateLegacyDocumentRefresh() constructor( private readonly databasePath: string, @@ -71,6 +73,7 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { assertProfileStateRevisionOnDisk(this.databasePath, this.profileId, revision) this.observedRevision = revision this.assertCurrentRevision() + this.legacyDocuments.startWriterThread(this.writableDatabase?.db) } get revision(): number { @@ -148,11 +151,13 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { this.readSerializedState() } const opened = this.openWritableDatabase() + this.legacyDocuments.catchUp(this.writableDatabase?.db) this.observedRevision = writeProfileStateDomains(opened.db, { expectedRevision: this.observedRevision ?? 0, replacements, automationRunsAfter }).revision + this.legacyDocuments.afterWrite(() => this.writableDatabase?.db) } assertCurrentRevision(): void { @@ -199,6 +204,7 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { }) } const opened = this.openWritableDatabase() + this.legacyDocuments.catchUp(this.writableDatabase?.db) const currentRevision = readProfileStateRevision(opened.db) const complete = buildCompleteDocumentReplacements(opened.db, replacements) if (currentRevision === 0 || complete.length === 0) { @@ -219,6 +225,7 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { expectedRevision: this.observedRevision ?? currentRevision, replacements: complete }).revision + this.legacyDocuments.afterWrite(() => this.writableDatabase?.db) } scheduleBackup(): void { @@ -247,9 +254,11 @@ export class ProfileStateSqliteAuthority implements ProfileStateAuthority { return quarantineProfileStateDatabase(this.databasePath, this.profileId, quarantineRoot, reason) } - close(): void { + /** Only a graceful shutdown refreshes legacy rows; quarantine and maintenance closes must not write. */ + close(options: { refreshLegacyDocuments?: boolean } = {}): void { this.assertActive() this.backupRotation?.stop() + this.legacyDocuments.beforeClose(this.writableDatabase?.db, options.refreshLegacyDocuments) this.writableDatabase?.db.close() this.writableDatabase = undefined } diff --git a/src/main/persistence/profile-state/profile-state-worker-authority.test.ts b/src/main/persistence/profile-state/profile-state-worker-authority.test.ts index 60f3b09c580..c449dc4dfd7 100644 --- a/src/main/persistence/profile-state/profile-state-worker-authority.test.ts +++ b/src/main/persistence/profile-state/profile-state-worker-authority.test.ts @@ -2,6 +2,8 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { ProfileStateBackupRotation } from './profile-state-backup-rotation' import { ProfileStateWriteWorkerClient } from './profile-state-writer-worker-client' +import { openProfileStateDatabaseReadOnly } from './profile-state-database' +import { isRecord } from './profile-state-document-validation' import { createWorkerMaintenanceFixture, maintenanceBarrier @@ -17,6 +19,37 @@ vi.mock('../../ssh/config/ssh-config-parser', () => ({ })) describe('worker authority close admission', () => { + it('leaves the older-build copy alone for maintenance and brings it current on close', async () => { + const { authority, databaseFile, profileId } = await createWorkerMaintenanceFixture() + const v1 = '{"runtime:a":{"activeTabId":"t1"}}' + const v2 = '{"runtime:a":{"activeTabId":"t2"}}' + const v3 = '{"runtime:a":{"activeTabId":"t3"}}' + // What a build without per-entry storage reads. + const readAsOlderBuild = () => { + const { db } = openProfileStateDatabaseReadOnly(databaseFile, profileId) + try { + const row: unknown = db + .prepare(`SELECT payload FROM profile_state_documents WHERE domain = ?`) + .get('workspaceSessionsByHostId') + return isRecord(row) ? row.payload : undefined + } finally { + db.close() + } + } + await authority.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v1 }]) + // Restarting the writer catches the older-build copy up to v1. + await (await authority.pauseForMaintenance()).resume() + await authority.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v2 }]) + + const maintenance = await authority.pauseForMaintenance() + expect(readAsOlderBuild()).toBe(v1) + await maintenance.resume() + await authority.writeSerializedDomains([{ domain: 'workspaceSessionsByHostId', payload: v3 }]) + + await authority.close() + expect(readAsOlderBuild()).toBe(v3) + }) + it('reports a failed maintenance resume without accepting a changed database', async () => { const notify = vi.fn() const { authority, peer } = await createWorkerMaintenanceFixture(undefined, notify) diff --git a/src/main/persistence/profile-state/profile-state-worker-authority.ts b/src/main/persistence/profile-state/profile-state-worker-authority.ts index 12f561fe621..49ba60ce74a 100644 --- a/src/main/persistence/profile-state/profile-state-worker-authority.ts +++ b/src/main/persistence/profile-state/profile-state-worker-authority.ts @@ -133,16 +133,17 @@ export class ProfileStateWorkerAuthority implements AsyncProfileStateAuthority { return this.backups.drain() } - close(): Promise { + /** By default a graceful shutdown, which also brings the downgrade copy current. */ + close(options: { refreshLegacyDocuments?: boolean } = {}): Promise { this.cancelIdleStop() this.writer?.stopAdmission() - this.closing ??= this.finishClose() + this.closing ??= this.finishClose(options.refreshLegacyDocuments !== false) return this.closing } async pauseForMaintenance(): Promise { this.assertWritable() - await this.close() + await this.close({ refreshLegacyDocuments: false }) const writer = this.writer const revision = writer ? writer.acknowledgedRevision : this.requireStoppedRevision() let consumed = false @@ -163,7 +164,7 @@ export class ProfileStateWorkerAuthority implements AsyncProfileStateAuthority { await this.writer.ready this.assertWritable() } catch (error) { - await this.close() + await this.close({ refreshLegacyDocuments: false }) throw error } } @@ -171,7 +172,7 @@ export class ProfileStateWorkerAuthority implements AsyncProfileStateAuthority { } async quarantineDatabase(quarantineRoot?: string, reason?: string) { - await this.close() + await this.close({ refreshLegacyDocuments: false }) return quarantineProfileStateDatabase( this.initialization.databasePath, this.initialization.profileId, @@ -180,12 +181,12 @@ export class ProfileStateWorkerAuthority implements AsyncProfileStateAuthority { ) } - private async finishClose(): Promise { + private async finishClose(refreshLegacyDocuments: boolean): Promise { this.backups.stop() // A stop already under way owns the writer; let it finish before deciding what is left to close. const writerClosed = this.stopping - ? this.stopping.then(() => this.writer?.close()) - : this.writer?.close() + ? this.stopping.then(() => this.writer?.close({ refreshLegacyDocuments })) + : this.writer?.close({ refreshLegacyDocuments }) const settled = await Promise.allSettled([this.backups.drain(), writerClosed]) for (const result of settled) { if (result.status === 'rejected') { @@ -309,7 +310,7 @@ export class ProfileStateWorkerAuthority implements AsyncProfileStateAuthority { return } this.stopping = writer - .close() + .close({ refreshLegacyDocuments: true }) .then(() => { if (this.writer === writer) { this.writer = null diff --git a/src/main/persistence/profile-state/profile-state-writer-connection.ts b/src/main/persistence/profile-state/profile-state-writer-connection.ts index b9a7214ae4e..d13f50dacdd 100644 --- a/src/main/persistence/profile-state/profile-state-writer-connection.ts +++ b/src/main/persistence/profile-state/profile-state-writer-connection.ts @@ -93,9 +93,9 @@ export class ProfileStateWriterConnection { this.draining = true } - close(): Promise { + close(options: { refreshLegacyDocuments?: boolean } = {}): Promise { this.stopAdmission() - this.closePromise ??= this.finishClose() + this.closePromise ??= this.finishClose(options.refreshLegacyDocuments === true) return this.closePromise } @@ -118,12 +118,12 @@ export class ProfileStateWriterConnection { return this.thread?.exited ?? true } - private async finishClose(): Promise { + private async finishClose(refreshLegacyDocuments: boolean): Promise { await this.active?.promise.catch(() => {}) if (!this.failure && !this.didExit) { const closeRequestId = this.nextId try { - await this.dispatch({ command: 'close' }) + await this.dispatch({ command: 'close', refreshLegacyDocuments }) } finally { // Ownership requires observed exit, not timeout or termination request. await this.awaitExit(closeRequestId) diff --git a/src/main/persistence/profile-state/profile-state-writer-protocol.ts b/src/main/persistence/profile-state/profile-state-writer-protocol.ts index 28e6f8a6a4d..fc6dd1f262d 100644 --- a/src/main/persistence/profile-state/profile-state-writer-protocol.ts +++ b/src/main/persistence/profile-state/profile-state-writer-protocol.ts @@ -18,7 +18,9 @@ export type ProfileStateWriterCommand = replacements: readonly ProfileStateDomainReplacement[] runPayloads: readonly string[] } - | { command: 'assert-revision' | 'close' } + | { command: 'assert-revision' } + /** Only a graceful shutdown refreshes the downgrade copy; maintenance and quarantine must not write. */ + | { command: 'close'; refreshLegacyDocuments?: boolean } | { command: 'export-json' | 'export-latest'; targetPath: string } export type ProfileStateWriterRequest = ProfileStateWriterCommand & { id: number } @@ -71,8 +73,12 @@ export function isProfileStateWriterRequest(value: unknown): value is ProfileSta case 'write-state': return value.payload instanceof Uint8Array case 'assert-revision': - case 'close': return true + case 'close': + return ( + value.refreshLegacyDocuments === undefined || + typeof value.refreshLegacyDocuments === 'boolean' + ) case 'export-json': case 'export-latest': return typeof value.targetPath === 'string' && value.targetPath.length > 0 diff --git a/src/main/persistence/profile-state/profile-state-writer-worker-entry.ts b/src/main/persistence/profile-state/profile-state-writer-worker-entry.ts index 1ec8f826549..b037ad87952 100644 --- a/src/main/persistence/profile-state/profile-state-writer-worker-entry.ts +++ b/src/main/persistence/profile-state/profile-state-writer-worker-entry.ts @@ -71,7 +71,7 @@ async function execute(request: ProfileStateWriterRequest): Promise