From 9130ef825711be15ed8443a6f0dda2cd4ffb8b2f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 13 Nov 2026 03:02:40 -0800 Subject: [PATCH] fix(native-chat): retire certified legacy journals after the grace period --- .github/workflows/journal-file-lock.yml | 30 ++ config/electron-builder.config.cjs | 20 + .../native-build/build-journal-file-lock.mjs | 66 +++ .../prebuilds/build-orcad-prebuilds.mjs | 4 + .../orcad-prebuild-slot-contents.mjs | 2 + .../orcad-windows-prebuild-cache.mjs | 2 + .../orcad-windows-prebuild-cache.test.mjs | 3 +- native/journal-file-lock/.gitignore | 1 + native/journal-file-lock/main.c | 73 +++ .../database/journal-sql-database.ts | 1 + .../journal-legacy-file-lock.test.ts | 88 ++++ .../journal-legacy-file-lock.ts | 77 +++ .../journal-legacy-retirement-files.ts | 137 ++++++ .../journal-legacy-retirement-state.ts | 106 ++++ .../journal-legacy-retirement.test.ts | 453 ++++++++++++++++++ .../journal-legacy-retirement.ts | 303 ++++++++++++ .../journal-legacy-source-proof.ts | 252 ++++++++++ .../journal-sql-background-maintenance.ts | 21 +- .../journal-sql-batch.ts | 25 +- .../journal-sql-maintenance.ts | 28 ++ .../journal-sql-owner.test.ts | 18 + .../journal-sql-reader-entry.ts | 13 + .../journal-sql-reader-pool.ts | 53 ++ .../journal-sql-reader-protocol.ts | 3 + .../journal-sql-worker-entry.ts | 30 +- .../instance-lock/orcad-instance-lock-hold.ts | 21 + src/shared/orcad-artifacts.ts | 5 +- 27 files changed, 1824 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/journal-file-lock.yml create mode 100644 config/scripts/native-build/build-journal-file-lock.mjs create mode 100644 native/journal-file-lock/.gitignore create mode 100644 native/journal-file-lock/main.c create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-file-lock.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-retirement-files.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-retirement-state.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-retirement.test.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-retirement.ts create mode 100644 src/main/native-chat/agent-session-journal/journal-legacy-source-proof.ts diff --git a/.github/workflows/journal-file-lock.yml b/.github/workflows/journal-file-lock.yml new file mode 100644 index 00000000000..527b51f3c47 --- /dev/null +++ b/.github/workflows/journal-file-lock.yml @@ -0,0 +1,30 @@ +name: Journal file lock +on: + pull_request: + paths: + - 'native/journal-file-lock/**' + - 'config/scripts/native-build/build-journal-file-lock.mjs' + - 'src/main/native-chat/agent-session-journal/journal-legacy*' + - '.github/workflows/journal-file-lock.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + lock: + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: ./.github/actions/install-node-dependencies + with: + native-runtime: node + node-version: '24' + - run: node config/scripts/native-build/build-journal-file-lock.mjs + - run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts + env: + ORCA_BACKGROUND_LAUNCH: '1' diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index 95b4b45eddd..ed740f12207 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -166,6 +166,11 @@ const emojiShortcodeDatasetResource = { to: 'node_modules/emojibase-data/en/shortcodes/emojibase.json' } const commonExtraResources = [ + { + from: 'node_modules/node-pty/build/Release', + to: 'bin', + filter: ['journal-file-lock', 'journal-file-lock.exe'] + }, relayExtraResource, orcadTemplateExtraResource, orcadTemplateNodeModulesExtraResource, @@ -408,6 +413,16 @@ module.exports = { if (context.electronPlatformName === 'linux') { ensureWaylandClipboard(context.arch) } + return import('./scripts/native-build/build-journal-file-lock.mjs').then( + ({ buildJournalFileLock }) => { + const lockName = + context.electronPlatformName === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock' + buildJournalFileLock( + join(__dirname, '..', 'node_modules', 'node-pty', 'build', 'Release', lockName), + context.arch === 3 ? 'arm64' : 'x64' + ) + } + ) }, afterPack: async (context) => { const resourcesDir = @@ -537,6 +552,11 @@ module.exports = { }) chmodUnixCliLaunchers(resourcesDir, context.electronPlatformName) if (context.electronPlatformName === 'darwin') { + await signMacStandaloneHelper( + join(resourcesDir, 'bin', 'journal-file-lock'), + 'journal file lock', + context.packager + ) await signMacComputerUseHelper(join(resourcesDir, 'Orca Computer Use.app'), context.packager) await signMacStandaloneHelper( join(resourcesDir, '..', 'MacOS', 'orca-notification-status'), diff --git a/config/scripts/native-build/build-journal-file-lock.mjs b/config/scripts/native-build/build-journal-file-lock.mjs new file mode 100644 index 00000000000..7e4c099977f --- /dev/null +++ b/config/scripts/native-build/build-journal-file-lock.mjs @@ -0,0 +1,66 @@ +import { mkdirSync } from 'node:fs' +import { dirname, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { + initializeWindowsNativeToolchain, + mergeWindowsCompilerEnvironment +} from './windows/initialize-windows-native-toolchain.mjs' +import { runProcessSync } from '@orca/process-host' + +export function buildJournalFileLock(output, arch = process.arch) { + const root = resolve(import.meta.dirname, '../../..') + mkdirSync(dirname(output), { recursive: true }) + const source = resolve(root, 'native/journal-file-lock/main.c') + const windows = process.platform === 'win32' + const args = windows + ? [ + '/nologo', + '/O2', + '/W4', + '/WX', + '/DUNICODE', + '/D_UNICODE', + source, + `/Fe:${output}`, + `/Fo:${output}.obj` + ] + : [ + '-O2', + '-Wall', + '-Wextra', + '-Werror', + ...(process.platform === 'darwin' ? ['-arch', arch === 'x64' ? 'x86_64' : arch] : []), + source, + '-o', + output + ] + if (process.platform === 'linux' && arch !== process.arch) { + throw new Error('native lock needs a target-native Linux builder') + } + if (windows && arch !== process.arch) { + throw new Error('native lock needs a target-native Windows builder') + } + const compilerEnvironment = windows ? initializeWindowsNativeToolchain() : undefined + const result = runProcessSync({ + program: windows ? 'cl.exe' : 'cc', + args, + cwd: root, + env: compilerEnvironment + ? mergeWindowsCompilerEnvironment(process.env, compilerEnvironment) + : process.env, + timeoutMs: 120000 + }) + if (result.code !== 0) { + throw new Error(`journal lock build failed: ${result.stderr}${result.stdout}`) + } + return output +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + buildJournalFileLock( + resolve( + 'native/journal-file-lock/.build', + process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock' + ) + ) +} diff --git a/config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs b/config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs index b3dcfd36328..b3d6aa7db9a 100644 --- a/config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs +++ b/config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { buildJournalFileLock } from '../../native-build/build-journal-file-lock.mjs' // Build patched node-pty for current platform/arch/libc so deployment needs no compiler; compat slots also build COMPAT_SLOT_ADDONS. // Require repo patch: GLIBC symbol pins plus no-as-needed retain libutil/libpthread and prevent #9902. // Use pinned, hash-verified Node headers at N-API 8; glibc slots target 2.28, optional compat 2.17, musl stays separate. @@ -254,6 +255,9 @@ async function build() { ? await compileCompatAddons({ slot, workDir: join(WORK_DIR, slot), nodeDir }) : [] + buildJournalFileLock( + join(buildDir, process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock') + ) rmSync(slotDir, { recursive: true, force: true }) const files = {} for (const [relative, source] of [ diff --git a/config/scripts/orcad-distribution/prebuilds/orcad-prebuild-slot-contents.mjs b/config/scripts/orcad-distribution/prebuilds/orcad-prebuild-slot-contents.mjs index c42dcde114a..1c237cff302 100644 --- a/config/scripts/orcad-distribution/prebuilds/orcad-prebuild-slot-contents.mjs +++ b/config/scripts/orcad-distribution/prebuilds/orcad-prebuild-slot-contents.mjs @@ -95,6 +95,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) { const conptyRuntime = windowsConptyRuntimeDir(nodePtyDir, arch) const upstream = join(nodePtyDir, 'prebuilds', `win32-${arch}`) return [ + ['journal-file-lock.exe', join(buildDir, 'journal-file-lock.exe')], ['conpty.node', join(buildDir, 'conpty.node')], ['conpty_console_list.node', join(upstream, 'conpty_console_list.node')], ['conpty/conpty.dll', join(conptyRuntime, 'conpty.dll')], @@ -104,6 +105,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) { } // macOS posix_spawns spawn-helper and builds it only there; Linux forks directly. return [ + ['journal-file-lock', join(buildDir, 'journal-file-lock')], ['pty.node', join(buildDir, 'pty.node')], ...(platform === 'darwin' ? [['spawn-helper', join(buildDir, 'spawn-helper')]] : []) ] diff --git a/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.mjs b/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.mjs index 886784b2251..e88da7cf07b 100644 --- a/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.mjs +++ b/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.mjs @@ -23,6 +23,8 @@ export const WINDOWS_PREBUILD_LOCKED_PACKAGES = ['node-pty', 'node-gyp'] const WINDOWS_PREBUILD_PACKAGE_SCRIPTS = ['build:orcad-prebuilds', 'build:packages'] export const WINDOWS_PREBUILD_CACHE_INPUTS = [ '.github/actions/prepare-orcad-prebuilds/action.yml', + 'native/journal-file-lock/main.c', + 'config/scripts/native-build/build-journal-file-lock.mjs', 'config/patches/node-pty@1.1.0.patch', 'src/shared/node-runtime-pin.ts', 'config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs', diff --git a/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.test.mjs b/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.test.mjs index fee158eac5c..98e96b0e380 100644 --- a/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.test.mjs +++ b/config/scripts/orcad-distribution/prebuilds/orcad-windows-prebuild-cache.test.mjs @@ -38,6 +38,7 @@ function fixture(arch = 'x64') { mkdirSync(join(sourceDir, 'third_party/conpty/1', `win10-${arch}`), { recursive: true }) const files = {} for (const [file, bytes] of [ + ['journal-file-lock.exe', peImage({ arch })], [ 'conpty.node', Buffer.concat([peImage({ arch }), CYGWIN_BREAKAWAY_MARKER, NATIVE_HOST_JOB_MARKER]) @@ -51,7 +52,7 @@ function fixture(arch = 'x64') { ]) { write(join(slotDir, file), bytes) files[file] = sha256Of(join(slotDir, file)) - if (file !== 'conpty.node' && !file.startsWith('conpty/')) { + if (file !== 'journal-file-lock.exe' && file !== 'conpty.node' && !file.startsWith('conpty/')) { write(join(sourceDir, 'prebuilds', slot, file), bytes) } if (file.startsWith('conpty/')) { diff --git a/native/journal-file-lock/.gitignore b/native/journal-file-lock/.gitignore new file mode 100644 index 00000000000..30bcfa4ed5c --- /dev/null +++ b/native/journal-file-lock/.gitignore @@ -0,0 +1 @@ +.build/ diff --git a/native/journal-file-lock/main.c b/native/journal-file-lock/main.c new file mode 100644 index 00000000000..fa2dd460b05 --- /dev/null +++ b/native/journal-file-lock/main.c @@ -0,0 +1,73 @@ +/* Holds the standard SQLite main-file lock without opening or modifying SQLite. */ +#ifdef _WIN32 +#define _WIN32_WINNT 0x0602 +#include +#include +#include +int wmain(int argc, wchar_t **argv) { + if (argc != 2) return 2; + HANDLE file = CreateFileW(argv[1], GENERIC_READ | DELETE, FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + if (file == INVALID_HANDLE_VALUE) return 3; + BY_HANDLE_FILE_INFORMATION info; + wchar_t fs[32], volume[MAX_PATH], final[32768]; + DWORD final_length = GetFinalPathNameByHandleW(file, final, 32768, FILE_NAME_NORMALIZED); + if (!GetFileInformationByHandle(file, &info) || info.nNumberOfLinks != 1 || + (info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT)) || + final_length == 0 || final_length >= 32768 || + !GetVolumePathNameW(final, volume, MAX_PATH) || GetDriveTypeW(volume) != DRIVE_FIXED || + !GetVolumeInformationByHandleW(file, NULL, 0, NULL, NULL, NULL, fs, 32) || + wcscmp(fs, L"NTFS") != 0) { + CloseHandle(file); return 4; + } + HANDLE named = CreateFileW(argv[1], 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); + BY_HANDLE_FILE_INFORMATION current; + int same = named != INVALID_HANDLE_VALUE && GetFileInformationByHandle(named, ¤t) && + current.dwVolumeSerialNumber == info.dwVolumeSerialNumber && + current.nFileIndexHigh == info.nFileIndexHigh && current.nFileIndexLow == info.nFileIndexLow; + if (named != INVALID_HANDLE_VALUE && !CloseHandle(named)) same = 0; + if (!same) { CloseHandle(file); return 4; } + puts("locked"); fflush(stdout); + while (getchar() != EOF) {} + return CloseHandle(file) ? 0 : 5; +} +#else +#include +#include +#include +#include +#include +#ifdef __APPLE__ +#include +#else +#include +#endif +int main(int argc, char **argv) { + if (argc != 2) return 2; + int fd = open(argv[1], O_RDWR | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) return 3; + struct stat info, named; + struct statfs volume; + int local = 0; + if (fstatfs(fd, &volume) == 0) { +#ifdef __APPLE__ + local = strcmp(volume.f_fstypename, "apfs") == 0 || strcmp(volume.f_fstypename, "hfs") == 0; +#else + local = volume.f_type == 0xef53 || volume.f_type == 0x58465342 || + volume.f_type == 0x9123683e || volume.f_type == 0x2fc12fc1; +#endif + } + struct flock lock = {0}; + lock.l_type = F_WRLCK; lock.l_whence = SEEK_SET; + lock.l_start = 0x40000000; lock.l_len = 512; + if (!local || fstat(fd, &info) || !S_ISREG(info.st_mode) || info.st_nlink != 1 || + fcntl(fd, F_SETLK, &lock) || lstat(argv[1], &named) || + info.st_dev != named.st_dev || info.st_ino != named.st_ino) { + close(fd); return 4; + } + puts("locked"); fflush(stdout); + while (getchar() != EOF) {} + return close(fd) == 0 ? 0 : 5; +} +#endif diff --git a/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts b/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts index 263ce9bc44d..0855be448e2 100644 --- a/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts +++ b/src/main/native-chat/agent-session-journal/database/journal-sql-database.ts @@ -61,6 +61,7 @@ export class JournalSqlDatabase { this.activity, options.onMaintenanceError ) + this.maintenance.ownLegacySource(stateDirectory, this.readers) } static async open( diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts b/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts new file mode 100644 index 00000000000..a0ec6d6a726 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts @@ -0,0 +1,88 @@ +import { captureJournalLegacySourceReceipt } from './database/journal-legacy-source-receipt' +import { existsSync, mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { spawnProcess } from '@orca/process-host' +import { afterEach, expect, it } from 'vitest' +import { acquireJournalLegacyFileLock, journalLegacyFileLockPath } from './journal-legacy-file-lock' +import { openJournalDatabase } from './database/journal-database' + +const cleanups: (() => void | Promise)[] = [] +afterEach(async () => { + for (const cleanup of cleanups.splice(0).toReversed()) { + await cleanup() + } +}) + +it.skipIf(!process.env.CI)( + 'excludes a real SQLite process without altering source bytes, then releases on EOF', + async () => { + expect(existsSync(journalLegacyFileLockPath())).toBe(true) + const root = mkdtempSync(join(tmpdir(), 'orca-journal-file-lock-')) + cleanups.push(() => rmSync(root, { recursive: true, force: true })) + const filename = join(root, 'source.db') + const { db } = openJournalDatabase(filename) + db.exec( + "CREATE TABLE fixture_history(value TEXT); INSERT INTO fixture_history VALUES('history never opened')" + ) + db.close() + const script = join(root, 'reader.cjs') + writeFileSync( + script, + "const{DatabaseSync}=require('node:sqlite');const db=new DatabaseSync(process.argv[2], {readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('ready');process.stdin.resume();process.stdin.on('end',()=>{db.close()});" + ) + const reader = spawnProcess({ + program: process.execPath, + args: [script, filename], + stdio: 'pipe' + }) + const exited = new Promise((resolve, reject) => { + reader.once('error', reject) + reader.once('exit', () => resolve()) + }) + cleanups.push(async () => { + reader.stdin.end() + await exited + }) + await new Promise((resolve, reject) => { + reader.stdout.once('data', () => resolve()) + reader.once('error', reject) + }) + const signal = new AbortController() + await expect(acquireJournalLegacyFileLock(filename, signal.signal)).rejects.toThrow() + reader.stdin.end() + await exited + const before = readFileSync(filename) + const identity = captureJournalLegacySourceReceipt(filename) + const lock = await acquireJournalLegacyFileLock(filename, signal.signal) + try { + expect(lock.held()).toBe(true) + expect(captureJournalLegacySourceReceipt(filename)).toEqual(identity) + const contender = spawnProcess({ + program: process.execPath, + args: [ + '-e', + "const{DatabaseSync}=require('node:sqlite');let db;try{db=new DatabaseSync(process.argv[1],{readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('opened')}catch{process.stdout.write('blocked')}finally{db?.close()}", + filename + ], + stdio: 'pipe' + }) + contender.stdin.end() + const output = await new Promise((resolve, reject) => { + let result = '' + contender.stdout.on('data', (chunk) => { + result += chunk.toString() + }) + contender.stderr.resume() + contender.once('error', reject) + contender.once('exit', () => resolve(result)) + }) + expect(output).toBe('blocked') + } finally { + await lock.close() + } + expect(readFileSync(filename)).toEqual(before) + const second = await acquireJournalLegacyFileLock(filename, signal.signal) + await second.close() + } +) diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.ts b/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.ts new file mode 100644 index 00000000000..c14058b0f2d --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-file-lock.ts @@ -0,0 +1,77 @@ +import { spawnProcess } from '@orca/process-host' +import { createRequire } from 'node:module' +import { existsSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { currentWorkerEntryLayout } from '../../worker-threads/worker-thread-entry-path' + +export function journalLegacyFileLockPath(): string { + const name = process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock' + const layout = currentWorkerEntryLayout(__dirname) + if (layout.isPackaged && layout.resourcesPath) { + return join(layout.resourcesPath, 'bin', name) + } + const require = createRequire(__filename) + const staged = join(dirname(require.resolve('node-pty/package.json')), 'build', 'Release', name) + return existsSync(staged) ? staged : resolve('native/journal-file-lock/.build', name) +} + +/** The helper never opens SQLite; closing stdin releases its single non-writing source handle. */ +export async function acquireJournalLegacyFileLock(filename: string, signal: AbortSignal) { + signal.throwIfAborted() + const child = spawnProcess({ + program: journalLegacyFileLockPath(), + args: [filename], + stdio: 'pipe' + }) + child.stdin.on('error', () => {}) + child.stderr.resume() + let exited = false + const done = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code) => { + exited = true + if (code === 0) { + resolve() + } else { + reject(new Error('legacy journal lock is unavailable')) + } + }) + }) + void done.catch(() => {}) + const release = () => child.stdin.end() + signal.addEventListener('abort', release, { once: true }) + if (signal.aborted) { + release() + } + try { + await new Promise((resolve, reject) => { + let output = '' + child.stdout.on('data', (chunk: Buffer) => { + output += chunk.toString() + if (output === 'locked\n' || output === 'locked\r\n') { + resolve() + } else if (output.length > 64) { + reject(new Error('invalid legacy lock acknowledgement')) + } + }) + child.once('error', reject) + child.once('exit', () => reject(new Error('legacy journal lock was refused'))) + }) + } catch (error) { + release() + await done.catch(() => {}) + signal.removeEventListener('abort', release) + throw error + } + return { + held: () => !exited && !signal.aborted, + close: async () => { + release() + try { + await done + } finally { + signal.removeEventListener('abort', release) + } + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-retirement-files.ts b/src/main/native-chat/agent-session-journal/journal-legacy-retirement-files.ts new file mode 100644 index 00000000000..c2da4fe8a47 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-retirement-files.ts @@ -0,0 +1,137 @@ +import { lstat, open, stat, unlink } from 'node:fs/promises' +import { lstatSync } from 'node:fs' +import { dirname } from 'node:path' +import { journalDomainDatabasePath } from './database/journal-domain-database' +import { + journalLegacyFileIdentity, + type JournalLegacyFileIdentity +} from './database/journal-legacy-source-receipt' + +export const LEGACY_COMPANION_SUFFIXES = ['-wal', '-shm'] as const +export type LegacyCompanionManifest = Partial< + Record<(typeof LEGACY_COMPANION_SUFFIXES)[number], JournalLegacyFileIdentity> +> +export type LegacyDestinationIdentity = { main: string; wal: string } +export function legacyDestinationMain(filename: string): string { + const file = lstatSync(filename, { bigint: true }) + if (!file.isFile() || file.nlink !== 1n || file.ino === 0n) { + throw new Error('invalid journal destination identity') + } + return `${file.dev}:${file.ino}` +} +export function legacyDestinationIdentity(filename: string): LegacyDestinationIdentity { + return { main: legacyDestinationMain(filename), wal: legacyDestinationMain(`${filename}-wal`) } +} + +export async function legacyFilePresent(filename: string): Promise { + try { + await lstat(filename) + return true + } catch (error) { + if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') { + return false + } + throw error + } +} + +export async function syncLegacyDirectory(directory: string): Promise { + if (process.platform === 'win32') { + return + } + // Ancestors are synced bottom-up so a newly created profile directory is covered too. + for (;;) { + const handle = await open(directory, 'r') + try { + await handle.sync() + } finally { + await handle.close() + } + const parent = dirname(directory) + if (parent === directory) { + return + } + directory = parent + } +} + +export async function syncLegacyDestination(directory: string, source: string): Promise { + const domain = journalDomainDatabasePath(directory) + const sourceFile = await stat(source, { bigint: true }) + for (const name of [domain, `${domain}-wal`]) { + const file = await lstat(name, { bigint: true }) + if ( + !file.isFile() || + file.dev !== sourceFile.dev || + file.ino === sourceFile.ino || + file.nlink !== 1n + ) { + throw new Error('journal destination volume or identity changed') + } + } + if (process.platform === 'win32') { + const sourceDevice = sourceFile.dev + for (const name of [domain, `${domain}-wal`]) { + const handle = await open(name, 'r+') + try { + if ((await handle.stat({ bigint: true })).dev !== sourceDevice) { + throw new Error('journal destination volume changed') + } + await handle.sync() + } finally { + await handle.close() + } + } + } + await syncLegacyDirectory(directory) +} + +export async function removeLegacyCompanions( + directory: string, + filename: string, + manifest: LegacyCompanionManifest | undefined, + boundary: () => Promise, + ownsProfile: () => boolean, + signal: AbortSignal +): Promise { + if (!manifest || !ownsProfile() || signal.aborted) { + return false + } + for (const suffix of LEGACY_COMPANION_SUFFIXES) { + if ( + !(await boundary()) || + !ownsProfile() || + (await legacyFilePresent(filename)) || + (await legacyFilePresent(`${filename}-journal`)) + ) { + return false + } + const name = filename + suffix + if (!(await legacyFilePresent(name))) { + continue + } + const expected = manifest[suffix] + const current = await lstat(name) + if ( + !current.isFile() || + !expected || + JSON.stringify(journalLegacyFileIdentity(name)) !== JSON.stringify(expected) || + (suffix === '-wal' && expected.size !== '0') + ) { + throw new Error('legacy companion changed; retaining it') + } + await unlink(name) + } + if ( + (await legacyFilePresent(filename)) || + ( + await Promise.all( + LEGACY_COMPANION_SUFFIXES.map((suffix) => legacyFilePresent(filename + suffix)) + ) + ).some(Boolean) + ) { + return false + } + await syncLegacyDirectory(directory) + return true +} diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-retirement-state.ts b/src/main/native-chat/agent-session-journal/journal-legacy-retirement-state.ts new file mode 100644 index 00000000000..bf30e582c95 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-retirement-state.ts @@ -0,0 +1,106 @@ +import type { LegacyCompanionManifest } from './journal-legacy-retirement-files' + +export const LEGACY_RETIREMENT_WEEK = 7 * 24 * 60 * 60 * 1000 +export type Retirement = { + version: 1 + proof: string + proofId: string + completedAt: number + process: string + credit: number + creditRevision: number + lastWall: number + phase: 'grace' | 'prepared' | 'complete' + manifest?: LegacyCompanionManifest +} + +export function decodeRetirement(value: string | null): Retirement | null { + if (!value) { + return null + } + const data: Retirement = JSON.parse(value) + if ( + data.version !== 1 || + typeof data.proof !== 'string' || + typeof data.proofId !== 'string' || + typeof data.process !== 'string' || + !['grace', 'prepared', 'complete'].includes(data.phase) || + !Number.isFinite(data.credit) || + data.credit < 0 || + data.credit > LEGACY_RETIREMENT_WEEK || + !Number.isSafeInteger(data.creditRevision) || + data.creditRevision < 0 + ) { + throw new Error('invalid legacy retirement receipt') + } + return data +} + +export function advanceLegacyRetirementClock( + state: Retirement, + now: number, + elapsed: number, + processToken: string +): Retirement { + if ( + !Number.isFinite(state.completedAt) || + !Number.isFinite(state.lastWall) || + state.completedAt < 0 || + state.lastWall < 0 || + state.completedAt > now || + state.lastWall > now + ) { + return { + ...state, + completedAt: now, + lastWall: now, + credit: 0, + process: processToken, + phase: 'grace' + } + } + return { + ...state, + credit: Math.min( + LEGACY_RETIREMENT_WEEK, + state.credit + Math.max(0, Number.isFinite(elapsed) ? elapsed : 0) + ), + lastWall: now + } +} + +/** Absolute revisions discard ambiguous saved credit instead of counting one interval twice. */ +export class JournalLegacyGraceClock { + private anchor?: bigint + private revision?: string + private pending = 0 + + observe(state: Retirement, now: number, processToken: string): Retirement { + const revision = `${state.proofId}:${state.creditRevision}` + if (this.revision !== revision) { + this.pending = 0 + this.anchor = undefined + this.revision = revision + } + const mono = process.hrtime.bigint() + if (this.anchor !== undefined && mono >= this.anchor) { + this.pending += Number(mono - this.anchor) / 1e6 + } + this.anchor = mono + return advanceLegacyRetirementClock(state, now, this.pending, processToken) + } + + persisted(state: Retirement): void { + this.revision = `${state.proofId}:${state.creditRevision}` + this.pending = 0 + this.anchor = process.hrtime.bigint() + } + + pause(): void { + const mono = process.hrtime.bigint() + if (this.anchor !== undefined && mono >= this.anchor) { + this.pending += Number(mono - this.anchor) / 1e6 + } + this.anchor = undefined + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-retirement.test.ts b/src/main/native-chat/agent-session-journal/journal-legacy-retirement.test.ts new file mode 100644 index 00000000000..e01ed35b895 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-retirement.test.ts @@ -0,0 +1,453 @@ +import { + mkdtempSync, + rmSync, + existsSync, + writeFileSync, + readFileSync, + statSync, + readdirSync, + unlinkSync, + statfsSync +} from 'node:fs' +import { join, basename } from 'node:path' +import { tmpdir } from 'node:os' +import { afterEach, expect, it, vi } from 'vitest' +import Database from '../../sqlite/sync-database' +import { openJournalDatabase } from './database/journal-database' +import { + initializeJournalDomainDatabase, + openJournalDomainDatabase +} from './database/journal-domain-database' +import { journalLegacyFileIdentity } from './database/journal-legacy-source-receipt' +import { JournalSqlDatabase } from './database/journal-sql-database' +import { + journalSqlWorkerPath, + journalSqlReaderPath +} from './journal-sql-worker-bundle.test-fixture' +import { message } from './journal-sql-message.test-fixture' +import { journalDatabasePath } from './database/journal-host-database' +import { JournalSqlBatch } from './journal-sql-batch' +import { JournalLegacyRetirement } from './journal-legacy-retirement' +import { + advanceLegacyRetirementClock, + JournalLegacyGraceClock +} from './journal-legacy-retirement-state' +import { readJournalLegacyProofPage } from './journal-legacy-source-proof' +import { applyJournalSqlMaintenance, type JournalSqlMaintenance } from './journal-sql-maintenance' +import { runJournalTransaction } from './database/journal-database' +import type { JournalSqlReaderPool } from './journal-sql-reader-pool' +import { journalDomainImportState } from './journal-domain-import-rows' +import { prepareJournalDomainImport } from './journal-domain-import' +import { journalSqlRetryRows } from './journal-sql-retry-rows.test-fixture' +import { insertJournalRow, publishJournalSessionEpoch } from './durable-rows/journal-row-table' + +const cleanup: (() => void)[] = [] +afterEach(() => { + vi.restoreAllMocks() + for (const close of cleanup.splice(0).toReversed()) { + close() + } +}) + +async function profile(seed?: (db: Database.Database) => void) { + const root = mkdtempSync(join(tmpdir(), 'orca-legacy-retirement-')) + cleanup.push(() => rmSync(root, { recursive: true, force: true })) + const filename = journalDatabasePath(root) + const { db: legacy } = openJournalDatabase(filename) + runJournalTransaction(legacy, () => { + publishJournalSessionEpoch( + legacy, + { sessionId: 'unopened', workspaceId: 'folder-without-git' }, + 'epoch' + ) + for (const row of journalSqlRetryRows(true)) { + insertJournalRow(legacy, 'unopened', row) + } + }) + seed?.(legacy) + legacy.close() + const { db } = openJournalDomainDatabase(await initializeJournalDomainDatabase(root)) + let closed = false + const closeDomain = () => { + if (!closed) { + db.close() + closed = true + } + } + cleanup.push(closeDomain) + let original: Database.Database | undefined + const source = () => + (original ??= new Database(filename, { readonly: true, fileMustExist: true })) + cleanup.push(() => original?.close()) + const batch = new JournalSqlBatch(db) + const writer = { + maintain: async (operation: JournalSqlMaintenance) => { + const result = await batch.run( + [operation], + (unit) => applyJournalSqlMaintenance(db, unit), + operation.kind === 'legacy-retirement-save' && operation.durable ? 'retirement' : 'normal' + ) + const outcome = result.outcomes[0]! + if (!outcome.committed) { + throw outcome.error + } + return outcome.value + } + } + const readers = { + legacyProof: async (cursor) => readJournalLegacyProofPage(source(), cursor, db), + importState: async (id) => journalDomainImportState(db, id), + recycleSourceReaders: async () => { + original?.close() + original = undefined + return new AbortController() + }, + releaseSourceReaders: () => {} + } satisfies Pick< + JournalSqlReaderPool, + 'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders' + > + const conversions = { + ensure: async (id: string) => { + if (journalDomainImportState(db, id).state === 'ready') { + return 'epoch' + } + const imported = prepareJournalDomainImport(source(), id) + if (!imported) { + return null + } + for (const page of imported.pages) { + await writer.maintain({ kind: 'import-page', target: imported, page }) + } + return String(await writer.maintain({ kind: 'import-finish', header: imported })) + } + } + const errors: unknown[] = [] + const lock = async () => ({ held: () => true, close: async () => {} }) + const retirement = new JournalLegacyRetirement( + root, + writer, + readers, + conversions, + async () => true, + (error) => errors.push(error), + () => true, + lock + ) + const receiptState = () => + JSON.parse( + String( + db + .prepare( + "SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'" + ) + .get()!.value + ) + ) + const saveState = (state: unknown) => + db + .prepare( + "UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'" + ) + .run(JSON.stringify(state)) + const arm = () => { + const row = db + .prepare( + "SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'" + ) + .get() + const state = JSON.parse(String(row!.value)) + state.credit = 604800000 + state.process = 'prior-process' + db.prepare( + "UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'" + ).run(JSON.stringify(state)) + } + return { + root, + filename, + db, + retirement, + errors, + arm, + source, + receiptState, + saveState, + writer, + closeDomain, + recycle: readers.recycleSourceReaders + } +} + +it('accounts for an unopened folder chat, removes only the certified file and keeps current history', async () => { + const test = await profile() + const allocated = () => ({ + files: Object.fromEntries( + readdirSync(test.root).map((name) => { + const file = statSync(join(test.root, name)) + return [name, { logical: Number(file.size), allocated: Number(file.blocks) * 512 }] + }) + ), + freeBytes: Number(statfsSync(test.root).bfree) * Number(statfsSync(test.root).bsize) + }) + const before = allocated() + const proofStart = performance.now() + await test.retirement.run() + expect(test.errors).toEqual([]) + expect(existsSync(test.filename)).toBe(true) + const proofMs = performance.now() - proofStart + const proof = test.receiptState() + test.saveState({ ...proof, credit: 604800000 }) + await test.retirement.run() + expect(existsSync(test.filename)).toBe(true) + test.arm() + const deletionStart = performance.now() + await test.retirement.run() + expect(test.errors).toEqual([]) + expect(existsSync(test.filename)).toBe(false) + expect( + test.db + .prepare( + "SELECT payload_json FROM journal_items WHERE session_id='unopened' AND payload_json LIKE '%compact%' LIMIT 1" + ) + .get() + ).toBeDefined() + await test.retirement.run() + expect(existsSync(test.filename)).toBe(false) + test.db.pragma('wal_checkpoint(TRUNCATE)') + test.closeDomain() + const after = allocated() + const total = (files: typeof before.files) => + Object.values(files).reduce((sum, value) => sum + value.allocated, 0) + console.info('synthetic allocated bytes after checkpoint and handle closure', { + before, + after, + grossLegacyRemoved: before.files[basename(test.filename)]!.allocated, + netRemoved: total(before.files) - total(after.files), + proofMs, + retireAndCheckpointMs: performance.now() - deletionStart + }) + const options = { + writerPath: journalSqlWorkerPath(), + readerPath: journalSqlReaderPath(), + onMaintenanceError: () => {} + } + const current = await JournalSqlDatabase.open(test.root, options) + try { + const { target } = await current.openSession({ + sessionId: 'new-current-chat', + workspaceId: 'folder-without-git', + hostId: 'synthetic-host', + agent: 'codex', + providerHandle: null + }) + const step = message('saved after legacy retirement') + await current.write({ + target, + savedAt: 100, + operation: { + kind: 'steps', + steps: [ + { + kind: 'resolved-item', + reservedBytes: 1024, + resolution: { kind: 'text-snapshot', identity: step.identity, body: step.body }, + options: step.options + } + ] + } + }) + } finally { + await current.close() + } + const reopened = await JournalSqlDatabase.open(test.root, options) + try { + expect( + await reopened.readers.domainHistory({ + sessionId: 'new-current-chat', + direction: 'after', + limit: 10 + }) + ).toMatchObject({ + page: { items: [{ body: { blocks: [{ text: 'saved after legacy retirement' }] } }] } + }) + expect(existsSync(test.filename)).toBe(false) + } finally { + await reopened.close() + } +}) + +it('retains downgrade writes, a rollback journal and a nonempty WAL without changing their bytes', async () => { + for (const kind of ['append', 'journal', 'wal'] as const) { + const test = await profile() + await test.retirement.run() + test.arm() + if (kind === 'append') { + const old = new Database(test.filename) + old + .prepare('INSERT INTO journal_sessions VALUES(?,?,?)') + .run('old-new-chat', 'folder', 'new-epoch') + const savedItem = journalSqlRetryRows(true).find((row) => row.kind === 'item')! + insertJournalRow(old, 'unopened', { + ...savedItem, + seq: journalSqlRetryRows(true).length + 1, + itemId: 'downgraded-text', + body: { + kind: 'message', + role: 'user', + blocks: [{ type: 'text', text: 'saved by the downgraded build' }] + } + }) + insertJournalRow(old, 'old-new-chat', { + ...journalSqlRetryRows(true)[0]!, + epoch: 'new-epoch', + seq: 1 + }) + old.close() + } else { + writeFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'owed recovery bytes') + } + const before = readFileSync(test.filename) + await test.retirement.run() + expect(existsSync(test.filename)).toBe(true) + expect(readFileSync(test.filename)).toEqual(before) + if (kind === 'append') { + expect( + test + .source() + .prepare("SELECT session_id FROM journal_sessions WHERE session_id='old-new-chat'") + .get() + ).toBeDefined() + expect( + test + .source() + .prepare( + "SELECT row_json FROM journal_rows WHERE session_id='unopened' ORDER BY seq DESC LIMIT 1" + ) + .get()!.row_json + ).toContain('saved by the downgraded build') + } else { + expect(readFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'utf8')).toBe( + 'owed recovery bytes' + ) + } + } +}) + +it('does not earn grace from forward wall-clock steps and re-stamps a future completion', () => { + const state = { + version: 1 as const, + proof: 'proof', + proofId: 'id', + completedAt: 100, + process: 'prior', + credit: 0, + creditRevision: 0, + lastWall: 100, + phase: 'grace' as const + } + expect(advanceLegacyRetirementClock(state, 691200100, 3600000, 'next').credit).toBe(3600000) + expect( + advanceLegacyRetirementClock( + { ...state, completedAt: 9000, credit: 604800000 }, + 1000, + 10, + 'next' + ) + ).toMatchObject({ completedAt: 1000, credit: 0, process: 'next' }) + + let mono = 0n + vi.spyOn(process.hrtime, 'bigint').mockImplementation(() => mono) + const clock = new JournalLegacyGraceClock() + clock.persisted(state) + mono = 3600000000000n + expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000) + clock.pause() + expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000) + const ambiguousCommit = { ...state, credit: 3600000, creditRevision: 1 } + mono = 7200000000000n + expect(clock.observe(ambiguousCommit, 691200100, 'next').credit).toBe(3600000) +}) + +it('retains queued-only text, recovery obligations and orphan history', async () => { + for (const kind of ['queue', 'repair', 'orphan', 'extra-profile-column'] as const) { + const test = await profile((legacy) => { + if (kind === 'queue') { + legacy.exec( + 'DELETE FROM journal_rows; INSERT INTO queued_messages(session_id,message_id,position,body_json,fingerprint,created_at,host_instance,state,hold_reason) VALUES(\'unopened\',\'held\',1,\'{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}\',\'f\',1,\'h\',\'waiting\',\'manual\')' + ) + } else if (kind === 'repair') { + legacy.exec("INSERT INTO journal_repairs VALUES('unopened','epoch',1,100)") + } else if (kind === 'extra-profile-column') { + legacy.exec( + "ALTER TABLE agent_session_records ADD COLUMN future_text TEXT; INSERT INTO agent_session_records VALUES('unopened','{}','saved future field')" + ) + } else { + legacy.exec("INSERT INTO journal_rows VALUES('orphan','old',1,1,'orphan user text')") + } + }) + if (kind === 'queue') { + const current = await JournalSqlDatabase.open(test.root, { + writerPath: journalSqlWorkerPath(), + readerPath: journalSqlReaderPath(), + onMaintenanceError: () => {} + }) + try { + await current.openSession({ + sessionId: 'unopened', + workspaceId: 'folder-without-git', + hostId: 'synthetic-host', + agent: 'codex', + providerHandle: null + }) + } finally { + await current.close() + } + } + const before = readFileSync(test.filename) + await test.retirement.run() + expect(test.errors.length).toBeGreaterThan(0) + expect(readFileSync(test.filename)).toEqual(before) + if (kind === 'queue') { + expect(test.source().prepare('SELECT body_json FROM queued_messages').get()).toEqual({ + body_json: + '{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}' + }) + } + } +}) + +it('resumes only the prepared empty companions and preserves a recreated main', async () => { + for (const recreate of [false, true]) { + const test = await profile() + await test.retirement.run() + await test.recycle() + const shm = `${test.filename}-shm` + writeFileSync(shm, 'derived index') + const state = { + ...test.receiptState(), + phase: 'prepared', + manifest: { + '-shm': journalLegacyFileIdentity(shm), + ...(existsSync(`${test.filename}-wal`) + ? { '-wal': journalLegacyFileIdentity(`${test.filename}-wal`) } + : {}) + } + } + test.saveState(state) + unlinkSync(test.filename) + if (recreate) { + writeFileSync(test.filename, 'new old-build history') + } + await test.retirement.run() + if (recreate) { + expect(readFileSync(shm, 'utf8')).toBe('derived index') + } + if (recreate) { + expect(readFileSync(test.filename, 'utf8')).toBe('new old-build history') + } else { + expect(test.errors).toEqual([]) + expect(existsSync(shm)).toBe(false) + } + } +}) diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-retirement.ts b/src/main/native-chat/agent-session-journal/journal-legacy-retirement.ts new file mode 100644 index 00000000000..b8932e156d8 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-retirement.ts @@ -0,0 +1,303 @@ +import { randomUUID } from 'node:crypto' +import { lstat, unlink } from 'node:fs/promises' +import { ownsOrcadProfileDirectory } from '../../orcad/instance-lock/orcad-instance-lock-hold' +import { journalDatabasePath } from './database/journal-host-database' +import { journalDomainDatabasePath } from './database/journal-domain-database' +import { + LEGACY_COMPANION_SUFFIXES as SUFFIXES, + legacyFilePresent as filePresent, + syncLegacyDirectory, + removeLegacyCompanions, + syncLegacyDestination, + legacyDestinationIdentity, + type LegacyCompanionManifest as Manifest, + type LegacyDestinationIdentity +} from './journal-legacy-retirement-files' +import { + LEGACY_RETIREMENT_WEEK, + decodeRetirement, + JournalLegacyGraceClock, + type Retirement +} from './journal-legacy-retirement-state' +import { + captureJournalLegacySourceReceipt, + journalLegacyFileIdentity +} from './database/journal-legacy-source-receipt' +import { acquireJournalLegacyFileLock } from './journal-legacy-file-lock' +import type { JournalSqlOwner } from './journal-sql-owner' +import type { JournalSqlReaderPool } from './journal-sql-reader-pool' +import type { JournalSqlConversions } from './journal-sql-conversions' +import { completeJournalLegacyProof } from './journal-legacy-source-proof' + +const WEEK = LEGACY_RETIREMENT_WEEK +const SAVE_INTERVAL = 6 * 60 * 60 * 1000 +const PROCESS_TOKEN = randomUUID() + +/** Existing maintenance owns proof, grace and one fixed unlink per yield; failures only retain data. */ +export class JournalLegacyRetirement { + private readonly filename: string + private readonly clock = new JournalLegacyGraceClock() + private lastSaved = 0 + private diagnosed?: string + private stopped = new AbortController() + + constructor( + private readonly directory: string, + private readonly writer: Pick, + private readonly readers: Pick< + JournalSqlReaderPool, + 'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders' + >, + private readonly conversions: Pick, + private readonly boundary: () => Promise, + private readonly report: (error: unknown) => void, + private readonly ownsProfile = () => ownsOrcadProfileDirectory(directory), + private readonly lock = acquireJournalLegacyFileLock + ) { + this.filename = journalDatabasePath(directory) + } + + stop(): void { + if (!this.stopped.signal.aborted) { + this.clock.pause() + this.stopped.abort() + } + } + + async run(): Promise { + try { + await this.attempt() + } catch (error) { + const reason = error instanceof Error ? error.message : String(error) + if (this.diagnosed !== reason) { + this.diagnosed = reason + this.report(error) + } + } + } + + async flushCredit(): Promise { + if (!this.stopped.signal.aborted) { + return + } + try { + const saved = await this.savedState() + const state = decodeRetirement(saved.retirement) + if ( + !state || + state.phase !== 'grace' || + state.proof !== saved.receipt || + !this.matches(state.proof) + ) { + return + } + const advanced = this.clock.observe(state, Date.now(), PROCESS_TOKEN) + if (advanced.completedAt === state.completedAt && advanced.credit > state.credit) { + await this.save(advanced, false) + } + } catch (error) { + this.report(error) + } + } + + private async savedState() { + const saved = await this.writer.maintain({ kind: 'legacy-retirement-state' }) + if ( + !saved || + typeof saved !== 'object' || + !('kind' in saved) || + saved.kind !== 'legacy-retirement-state' + ) { + throw new Error('legacy retirement state is unavailable') + } + return saved + } + + private async save( + state: Retirement, + durable: boolean, + destination?: LegacyDestinationIdentity + ): Promise { + state.creditRevision++ + if ( + (await this.writer.maintain({ + kind: 'legacy-retirement-save', + value: JSON.stringify(state), + durable, + destination + })) !== true + ) { + throw new Error('legacy retirement commit was not acknowledged') + } + this.lastSaved = Date.now() + this.clock.persisted(state) + } + + private matches(proof: string): boolean { + return JSON.stringify(captureJournalLegacySourceReceipt(this.filename)) === proof + } + + private prove(): Promise { + return completeJournalLegacyProof( + this.readers, + this.conversions, + this.writer, + this.boundary, + this.stopped.signal + ) + } + + private async attempt(): Promise { + if (this.stopped.signal.aborted) { + return + } + const saved = await this.savedState() + if (!saved.receipt) { + return + } + const receipt = JSON.parse(saved.receipt) + if ( + receipt.version !== 1 || + receipt.profileCopyInventoryVersion !== 1 || + receipt.disposition !== 'copied-source' || + receipt.wal?.kind !== 'empty' + ) { + return + } + const proof = saved.receipt + let state = decodeRetirement(saved.retirement) + if (state && state.proof !== saved.receipt) { + throw new Error('legacy retirement provenance changed') + } + if (state?.phase === 'complete') { + if (!(await filePresent(this.filename))) { + return + } + state = { ...state, phase: 'prepared' } + } + if (state?.phase === 'prepared' && !(await filePresent(this.filename))) { + await this.removeCompanions(state) + return + } + if (!this.matches(proof)) { + throw new Error('legacy source changed; retaining the original') + } + const now = Date.now() + if (!state) { + await this.prove() + if (!this.matches(proof)) { + throw new Error('legacy source changed during proof') + } + state = { + version: 1, + proof: saved.receipt, + proofId: randomUUID(), + completedAt: now, + process: PROCESS_TOKEN, + credit: 0, + creditRevision: 0, + lastWall: now, + phase: 'grace' + } + await this.save(state, true) + return + } + const advanced = this.clock.observe(state, now, PROCESS_TOKEN) + if (advanced.process !== state.process || advanced.completedAt !== state.completedAt) { + await this.prove() + if (!this.matches(proof)) { + return + } + await this.save(advanced, true) + return + } + state = advanced + if (state.credit < WEEK || state.process === PROCESS_TOKEN) { + if (now - this.lastSaved >= SAVE_INTERVAL) { + await this.save(state, false) + } + return + } + await this.prove() + if (!this.ownsProfile() || !this.matches(proof)) { + return + } + const recycled = await this.readers.recycleSourceReaders() + if (!recycled) { + return + } + const signal = AbortSignal.any([recycled.signal, this.stopped.signal]) + let lease: Awaited> | undefined + try { + lease = await this.lock(this.filename, signal) + const validate = () => { + if (!lease?.held() || signal.aborted || !this.ownsProfile() || !this.matches(proof)) { + throw new Error('legacy retirement lost exclusive ownership') + } + } + validate() + if (await filePresent(`${this.filename}-journal`)) { + throw new Error('legacy rollback journal retained') + } + const manifest: Manifest = {} + for (const suffix of SUFFIXES) { + if (await filePresent(this.filename + suffix)) { + const file = await lstat(this.filename + suffix) + if (!file.isFile()) { + throw new Error('legacy companion is not a regular file') + } + manifest[suffix] = journalLegacyFileIdentity(this.filename + suffix) + } + } + if (manifest['-wal'] && manifest['-wal'].size !== '0') { + throw new Error('legacy WAL still contains data') + } + const domain = journalDomainDatabasePath(this.directory) + const destination = legacyDestinationIdentity(domain) + await syncLegacyDestination(this.directory, this.filename) + if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) { + throw new Error('journal destination changed during sync') + } + state = { ...state, phase: 'prepared', manifest } + await this.save(state, true, destination) + validate() + if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) { + throw new Error('journal destination changed after prepare') + } + if (process.platform === 'win32') { + await lease.close() + lease = undefined + } + if (!this.ownsProfile() || signal.aborted || !this.matches(proof)) { + return + } + await unlink(this.filename) + await syncLegacyDirectory(this.directory) + await lease?.close() + lease = undefined + this.readers.releaseSourceReaders(recycled) + await this.removeCompanions(state) + } finally { + try { + await lease?.close() + } finally { + this.readers.releaseSourceReaders(recycled) + } + } + } + + private async removeCompanions(state: Retirement): Promise { + if ( + await removeLegacyCompanions( + this.directory, + this.filename, + state.manifest, + this.boundary, + this.ownsProfile, + this.stopped.signal + ) + ) { + await this.save({ ...state, phase: 'complete' }, true) + } + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-legacy-source-proof.ts b/src/main/native-chat/agent-session-journal/journal-legacy-source-proof.ts new file mode 100644 index 00000000000..e739d9409e2 --- /dev/null +++ b/src/main/native-chat/agent-session-journal/journal-legacy-source-proof.ts @@ -0,0 +1,252 @@ +import { readStoredQueuedMessageRow } from './queued-messages/queued-message-stored-row' +import type { JournalSqlReaderPool } from './journal-sql-reader-pool' +import type { JournalSqlConversions } from './journal-sql-conversions' +import type { JournalSqlOwner } from './journal-sql-owner' +import type Database from '../../sqlite/sync-database' + +const PROFILE_TABLES = new Set([ + 'agent_session_records', + 'agent_session_operations', + 'agent_session_retired_claim_keys', + 'agent_session_tabs', + 'agent_session_store_meta', + 'agent_session_command_receipts', + 'agent_session_attachment_claims', + 'agent_session_attachment_sweeps', + 'pr_chat_actions', + 'pr_check_events', + 'pr_event_deliveries', + 'pr_pending_heads', + 'pr_pending_actions', + 'pr_check_contexts', + 'sqlite_sequence' +]) +const CONTENT_TABLES = [ + 'journal_rows', + 'journal_outcome_generations', + 'journal_outcome_epochs', + 'journal_outcomes', + 'queued_messages' +] as const +const SOURCE_COLUMNS: Readonly> = { + journal_sessions: 'session_id workspace_id epoch', + journal_rows: 'session_id epoch seq ts row_json', + journal_outcome_generations: 'generation session_id epoch', + journal_outcome_epochs: 'session_id epoch generation format through_seq prefix_seq', + journal_outcomes: 'generation seq field entry_key format ordinal value_json deleted', + queued_messages: + 'session_id message_id position body_json fingerprint created_at host_instance state hold_reason returned_reason returned_rejection settled_at settled_by_op consumed_as carried_from queued_epoch queued_sequence options_json' +} +const RECOVERY_TABLES = ['journal_repairs', 'journal_imports', 'journal_set_aside'] +export type LegacyProofCursor = { table: number; after: string | null } +export type LegacySourceSession = { + sessionId: string + workspaceId: string + epoch: string + empty: boolean +} +export type LegacyProofPage = { next: LegacyProofCursor | null; sessions: LegacySourceSession[] } + +/** Every saved content row must belong to the current catalog before any source file can retire. */ +export function readJournalLegacyProofPage( + db: Database.Database, + cursor: LegacyProofCursor, + destination: Database.Database +): LegacyProofPage { + const tables = new Set( + db + .prepare("SELECT name FROM sqlite_schema WHERE type='table'") + .all() + .map((row) => String(row.name)) + ) + if (cursor.table === 0 && cursor.after === null) { + const version = Number(db.pragma('user_version', { simple: true })) + if (version !== 3 && version !== 4) { + throw new Error('unsupported legacy source inventory') + } + for (const table of tables) { + const names = + PROFILE_TABLES.has(table) && table !== 'sqlite_sequence' + ? destination + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((column) => column.name) + : SOURCE_COLUMNS[table]?.split(' ') + if (names) { + const columns = new Set(names) + if ( + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .some((column) => !columns.has(column.name)) + ) { + throw new Error('legacy source has unaccounted columns') + } + } + + if ( + PROFILE_TABLES.has(table) || + CONTENT_TABLES.includes(table as (typeof CONTENT_TABLES)[number]) || + table === 'journal_sessions' + ) { + continue + } + const quoted = `"${table.replaceAll('"', '""')}"` + if (db.prepare(`SELECT 1 FROM ${quoted} LIMIT 1`).get()) { + throw new Error('legacy source has unaccounted recovery data') + } + if (!RECOVERY_TABLES.includes(table)) { + throw new Error('legacy source has an unknown table') + } + } + } + if (cursor.table < CONTENT_TABLES.length) { + const table = CONTENT_TABLES[cursor.table]! + if (!tables.has(table)) { + return { next: { table: cursor.table + 1, after: null }, sessions: [] } + } + const columns = + table === 'journal_outcomes' + ? 'generation' + : table === 'queued_messages' + ? `session_id,${ + db + .prepare('PRAGMA table_info(queued_messages)') + .all() + .some((column) => column.name === 'queued_epoch') + ? 'queued_epoch' + : 'NULL' + } AS epoch` + : table === 'journal_rows' + ? 'session_id,epoch' + : 'session_id,epoch,generation' + const statement = db.prepare( + `SELECT rowid AS rowid,${columns} FROM ${table} ${cursor.after === null ? '' : 'WHERE rowid>?'} ORDER BY rowid LIMIT 128` + ) + statement.setReadBigInts(true) + const rows = statement.all(...(cursor.after === null ? [] : [BigInt(cursor.after)])) + for (const row of rows) { + if ( + table === 'queued_messages' && + !readStoredQueuedMessageRow( + db.prepare('SELECT * FROM queued_messages WHERE rowid=?').get(row.rowid!) + ) + ) { + throw new Error('legacy source has unreadable queued text') + } + + const session = + table === 'journal_outcomes' + ? db + .prepare( + 'SELECT session_id,epoch FROM journal_outcome_generations WHERE generation=?' + ) + .get(row.generation!) + : row + if ( + !session || + !db + .prepare( + `SELECT 1 FROM journal_sessions WHERE session_id=?${ + table === 'queued_messages' ? '' : ' AND epoch=?' + }` + ) + .get( + session.session_id!, + ...(table === 'queued_messages' && session.epoch == null ? [] : [session.epoch!]) + ) + ) { + throw new Error('legacy source has orphan content') + } + if (table.startsWith('journal_outcome')) { + const generation = row.generation + if ( + !db + .prepare( + 'SELECT 1 FROM journal_outcome_epochs WHERE session_id=? AND epoch=? AND generation=?' + ) + .get(session.session_id!, session.epoch!, generation!) + ) { + throw new Error('legacy source has an orphan outcome generation') + } + } + } + return { + next: rows.length + ? { table: cursor.table, after: String(rows.at(-1)!.rowid) } + : { table: cursor.table + 1, after: null }, + sessions: [] + } + } + const rows = db + .prepare( + `SELECT session_id,workspace_id,epoch FROM journal_sessions ${cursor.after === null ? '' : 'WHERE session_id>?'} ORDER BY session_id LIMIT 32` + ) + .all(...(cursor.after === null ? [] : [cursor.after])) + const sessions = rows.map((row) => { + const id = String(row.session_id) + const hasRows = !!db.prepare('SELECT 1 FROM journal_rows WHERE session_id=? LIMIT 1').get(id) + const hasQueue = + tables.has('queued_messages') && + !!db.prepare('SELECT 1 FROM queued_messages WHERE session_id=? LIMIT 1').get(id) + if (!hasRows && hasQueue) { + throw new Error('legacy queued-only history still requires its source') + } + const empty = + !hasRows && + !hasQueue && + (!tables.has('journal_outcome_epochs') || + !db.prepare('SELECT 1 FROM journal_outcome_epochs WHERE session_id=? LIMIT 1').get(id)) + return { sessionId: id, workspaceId: String(row.workspace_id), epoch: String(row.epoch), empty } + }) + return { + next: rows.length ? { table: cursor.table, after: String(rows.at(-1)!.session_id) } : null, + sessions + } +} + +export async function completeJournalLegacyProof( + readers: Pick, + conversions: Pick, + writer: Pick, + boundary: () => Promise, + signal: AbortSignal +): Promise { + let cursor: LegacyProofCursor | null = { table: 0, after: null } + while (cursor && !signal.aborted) { + if (!(await boundary())) { + throw new Error('legacy proof was interrupted') + } + const page = await readers.legacyProof(cursor) + for (const source of page.sessions) { + await conversions.ensure(source.sessionId, false) + if ((await readers.importState(source.sessionId)).state !== 'ready' && source.empty) { + await writer.maintain({ + kind: 'import-finish', + header: { + sessionId: source.sessionId, + workspaceId: source.workspaceId, + epoch: source.epoch, + head: { + lastSequence: 0, + highestFence: 0, + contextBoundary: 0, + lastActivityAt: 0, + derivedTurnItemId: null, + historyOrigin: null, + queueFailureBoundary: 0, + queuePauseMarks: { latestStop: null, resumedSequence: 0, reopenedSequence: 0 } + } + } + }) + } + if ((await readers.importState(source.sessionId)).state !== 'ready') { + throw new Error('legacy source still has unconverted or queued history') + } + } + cursor = page.next + } + if (signal.aborted) { + throw new Error('legacy proof was interrupted') + } +} diff --git a/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts index fce5f4b557e..46324ff4fad 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-background-maintenance.ts @@ -1,3 +1,4 @@ +import { JournalLegacyRetirement } from './journal-legacy-retirement' import { setImmediate, setTimeout as delay } from 'node:timers/promises' import type { JournalSqlOwner } from './journal-sql-owner' import type { JournalSqlReaderPool } from './journal-sql-reader-pool' @@ -30,6 +31,19 @@ export class JournalSqlBackgroundMaintenance { private readonly onError: (error: unknown, sessionId?: string) => void ) {} + private legacyRetirement?: JournalLegacyRetirement + + ownLegacySource(directory: string, readers: JournalSqlReaderPool): void { + this.legacyRetirement = new JournalLegacyRetirement( + directory, + this.writer, + readers, + this.conversions, + () => this.boundary(), + this.onError + ) + } + start(): Promise { if (this.stopped.signal.aborted) { return Promise.resolve() @@ -75,13 +89,15 @@ export class JournalSqlBackgroundMaintenance { } stop(): void { + this.legacyRetirement?.stop() this.stopped.abort() clearTimeout(this.timer) this.timer = undefined } - settled(): Promise { - return this.running ?? Promise.resolve() + async settled(): Promise { + await this.running + await this.legacyRetirement?.flushCredit() } private async boundary(): Promise { @@ -102,6 +118,7 @@ export class JournalSqlBackgroundMaintenance { private async run(): Promise { while (!this.stopped.signal.aborted && (this.discover || this.sweep || this.periodic)) { await this.runOrdinaryWork() + await this.legacyRetirement?.run() if (this.discover) { this.discover = false try { diff --git a/src/main/native-chat/agent-session-journal/journal-sql-batch.ts b/src/main/native-chat/agent-session-journal/journal-sql-batch.ts index e26798a3309..a68d819c5b7 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-batch.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-batch.ts @@ -37,7 +37,7 @@ export class JournalSqlBatch { async run( commands: readonly Command[], apply: (command: Command) => Value | Promise, - durability: 'normal' | 'full' = 'normal' + durability: 'normal' | 'full' | 'retirement' = 'normal' ): Promise> { if (this.failure) { throw new JournalSqlBatchError('not-dispatched', this.failure) @@ -47,13 +47,24 @@ export class JournalSqlBatch { return { outcomes } } const drain = commands.slice() + const pragmas = + durability === 'retirement' + ? ['synchronous', 'busy_timeout', 'fullfsync', 'checkpoint_fullfsync'].map( + (name) => [name, this.db.pragma(name, { simple: true })] as const + ) + : [] let began = false let committing = false let failed: JournalSqlBatchError | undefined try { await this.checkpoint?.beforeWrite() - if (durability === 'full') { + if (durability !== 'normal') { this.db.pragma('synchronous=FULL') + if (durability === 'retirement') { + this.db.pragma('busy_timeout=0') + this.db.pragma('fullfsync=ON') + this.db.pragma('checkpoint_fullfsync=ON') + } } this.db.exec('BEGIN IMMEDIATE') began = true @@ -89,9 +100,15 @@ export class JournalSqlBatch { this.failure = failed } } finally { - if (durability === 'full' && !this.db.isTransaction) { + if (durability !== 'normal' && !this.db.isTransaction) { try { - this.db.pragma('synchronous=NORMAL') + if (durability === 'retirement') { + for (const [name, value] of pragmas) { + this.db.pragma(`${name}=${Number(value)}`) + } + } else { + this.db.pragma('synchronous=NORMAL') + } } catch (cause) { failed ??= new JournalSqlBatchError('indeterminate', cause) this.failure = failed diff --git a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts index e7961c68bbf..2ae2f9ecc54 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-maintenance.ts @@ -1,3 +1,4 @@ +import type { LegacyDestinationIdentity } from './journal-legacy-retirement-files' import { rekeyWslDriveUncChats } from '../../wsl/filesystem/wsl-drive-unc-chat-rekey' import type { JournalForkStageOperation, JournalForkStageResult } from './journal-fork-stages' import { saveJournalDomainOutlineCache, type JournalOutlineCache } from './journal-domain-outline' @@ -26,6 +27,13 @@ import { installAgentSessionStoreRows } from '../../runtime/agent-session/record import type { JournalSqlStoredRecords } from './journal-sql-database-read' export type JournalSqlMaintenance = + | { kind: 'legacy-retirement-state' } + | { + kind: 'legacy-retirement-save' + value: string + durable: boolean + destination?: LegacyDestinationIdentity + } | { kind: 'records-install' } | { kind: 'wsl-workspace-rekey'; worktreeIds: readonly (readonly [string, string])[] } | JournalForkStageOperation @@ -44,6 +52,7 @@ export type JournalSqlMaintenance = export type JournalSqlProjectionState = JournalDomainImportState export type JournalSqlMaintenanceResult = + | { kind: 'legacy-retirement-state'; receipt: string | null; retirement: string | null } | { kind: 'records'; records: JournalSqlStoredRecords } | JournalForkStageResult | JournalSqlProjectionState @@ -59,6 +68,25 @@ export function applyJournalSqlMaintenance( operation: JournalSqlMaintenance ): JournalSqlMaintenanceResult { switch (operation.kind) { + case 'legacy-retirement-state': { + const value = (key: string) => + db + .prepare('SELECT value FROM agent_session_store_meta WHERE key=?') + .get(key) + ?.value?.toString() ?? null + return { + kind: 'legacy-retirement-state', + receipt: value('journal.legacy-source-copy.v1'), + retirement: value('journal.legacy-retirement.v1') + } + } + case 'legacy-retirement-save': + db.prepare('INSERT OR REPLACE INTO agent_session_store_meta(key,value) VALUES(?,?)').run( + 'journal.legacy-retirement.v1', + operation.value + ) + return true + case 'wsl-workspace-rekey': return rekeyWslDriveUncChats(db, operation.worktreeIds) case 'records-install': { diff --git a/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts b/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts index 83f5bff3a32..f3b185bca04 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-owner.test.ts @@ -207,3 +207,21 @@ it('correlates interleaved replies and resumes current bodies after writer closu }) expect(reset.page.items).toHaveLength(1) }) + +it('serves domain reads while source recycling yields to a new chat', async () => { + const { filename, root } = await fixture() + const pool = new JournalSqlReaderPool( + filename, + join(root, 'absent-legacy.db'), + journalSqlReaderPath() + ) + readers.push(pool) + await pool.importState('session') + const lease = await pool.recycleSourceReaders() + expect(lease).not.toBeNull() + const conversion = pool.prepareConversion('new-current-chat') + expect(await pool.importState('session')).toMatchObject({ state: 'ready' }) + pool.releaseSourceReaders(lease!) + expect(await conversion).toBeNull() + expect(await pool.importState('session')).toMatchObject({ state: 'ready' }) +}) diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts index 8d5c1952c56..bd319e80f2b 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reader-entry.ts @@ -1,3 +1,4 @@ +import { readJournalLegacyProofPage } from './journal-legacy-source-proof' import { JournalForkPreparation } from './journal-fork-preparation' import { parentPort, workerData } from 'node:worker_threads' import { existsSync } from 'node:fs' @@ -83,6 +84,7 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => { case 'database': reply({ type: 'database', id, result: readJournalSqlDatabase(db, operation.operation) }) break + case 'legacy-proof': case 'conversion-prepare': { // Each frozen-source read is its own short statement; no snapshot spans the restore. if (!original && existsSync(workerData.originalFilename)) { @@ -92,6 +94,17 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => { timeout: 5000 }) } + if (operation.kind === 'legacy-proof') { + if (!original) { + throw new Error('legacy source disappeared during proof') + } + reply({ + type: 'legacy-proof', + id, + page: readJournalLegacyProofPage(original, operation.cursor, db) + }) + break + } const prepared = original ? prepareJournalDomainImport(original, operation.sessionId) : null if (prepared) { const { pages, ...header } = prepared diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reader-pool.ts b/src/main/native-chat/agent-session-journal/journal-sql-reader-pool.ts index 1573acf75c0..8f4855dc431 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reader-pool.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reader-pool.ts @@ -1,3 +1,4 @@ +import type { LegacyProofCursor } from './journal-legacy-source-proof' import type { JournalSqlForkReader, JournalSqlConversionReader } from './journal-sql-reader-slot' import { ReaderSlot } from './journal-sql-reader-slot' import type { JournalForkPreparationInput } from './journal-fork-preparation' @@ -22,6 +23,8 @@ export class JournalSqlReaderPool { private closed = false private closing?: Promise private conversionSlot?: ReaderSlot + private sourceLease?: AbortController + private sourceRelease?: PromiseWithResolvers private forkSlot?: ReaderSlot private forkRelease?: PromiseWithResolvers @@ -99,12 +102,61 @@ export class JournalSqlReaderPool { } async prepareConversion(sessionId: string): Promise { + await this.releaseSourceLease() if (!this.conversionSlot || this.conversionSlot.failed) { this.conversionSlot = this.slot() } return this.conversionSlot.prepareConversion(sessionId) } + async legacyProof(cursor: LegacyProofCursor) { + await this.releaseSourceLease() + this.conversionSlot ??= this.slot() + const response = await this.conversionSlot.request({ kind: 'legacy-proof', cursor }) + if (response.type !== 'legacy-proof') { + throw new Error('legacy proof reader returned no page') + } + return response.page + } + + async recycleSourceReaders(): Promise { + if ( + this.closed || + this.sourceLease || + this.forkSlot || + this.slots.some((slot) => slot.pendingCount !== 0) + ) { + return null + } + const lease = new AbortController() + this.sourceLease = lease + this.sourceRelease = Promise.withResolvers() + const retired = this.slots.splice(0) + this.conversionSlot = undefined + try { + await Promise.all([...this.retiring, ...retired.map((slot) => slot.close())]) + return lease + } catch (error) { + this.releaseSourceReaders(lease) + throw error + } + } + + releaseSourceReaders(lease: AbortController): void { + if (this.sourceLease === lease) { + this.sourceLease = undefined + this.sourceRelease?.resolve() + this.sourceRelease = undefined + } + } + + private async releaseSourceLease(): Promise { + if (this.sourceLease) { + this.sourceLease.abort() + await this.sourceRelease?.promise + } + } + async importState(sessionId: string) { const response = await this.slot().request({ kind: 'import-state', sessionId }) if (response.type !== 'import-state') { @@ -159,6 +211,7 @@ export class JournalSqlReaderPool { } close(): Promise { + this.sourceLease?.abort() this.closed = true this.forkRelease?.resolve() return (this.closing ??= Promise.all([ diff --git a/src/main/native-chat/agent-session-journal/journal-sql-reader-protocol.ts b/src/main/native-chat/agent-session-journal/journal-sql-reader-protocol.ts index 18d17038ebb..0524758c4ae 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-reader-protocol.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-reader-protocol.ts @@ -1,3 +1,4 @@ +import type { LegacyProofCursor, LegacyProofPage } from './journal-legacy-source-proof' import type { JournalForkPreparationInput, JournalForkPreparationResult @@ -14,6 +15,7 @@ import type { JournalDomainHistoryRequest } from './journal-domain-history-page' import type { JournalDomainImportState } from './journal-domain-import-rows' export type JournalSqlReaderOperation = + | { kind: 'legacy-proof'; cursor: LegacyProofCursor } | { kind: 'fork-prepare'; input: JournalForkPreparationInput } | { kind: 'fork-next'; preparationId: number } | { kind: 'fork-release'; preparationId: number } @@ -27,6 +29,7 @@ export type JournalSqlReaderOperation = export type JournalSqlReaderRequest = { id: number; operation: JournalSqlReaderOperation } export type JournalSqlReaderReply = + | { type: 'legacy-proof'; id: number; page: LegacyProofPage } | { type: 'fork-prepared'; id: number } | { type: 'fork-next'; id: number; result: JournalForkPreparationResult } | { type: 'ready' } diff --git a/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts b/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts index 8b0cdb63c24..f59273bf3d1 100644 --- a/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts +++ b/src/main/native-chat/agent-session-journal/journal-sql-worker-entry.ts @@ -1,3 +1,4 @@ +import { legacyDestinationMain, legacyDestinationIdentity } from './journal-legacy-retirement-files' import { JournalForkStages } from './journal-fork-stages' import { mkdirSync } from 'node:fs' import { dirname } from 'node:path' @@ -34,6 +35,7 @@ let commands: JournalSqlCommand | undefined let forks: JournalForkStages | undefined let checkpoint: JournalCheckpointOwner | undefined let checkpointClosing: Promise | undefined +let destinationMain: string | undefined let readOnly = true let fault: unknown let closing = false @@ -48,6 +50,11 @@ async function open(): Promise { } const opened = openJournalDomainDatabase(workerData.filename) db = opened.db + try { + destinationMain = legacyDestinationMain(workerData.filename) + } catch { + /* Cleanup eligibility never gates opening a chat. */ + } readOnly = opened.readOnly forks = new JournalForkStages(db, stateDirectory) commands = new JournalSqlCommand(db, stateDirectory, forks) @@ -126,6 +133,16 @@ async function drain(): Promise { const checkpoint = forks!.checkpoint() try { if (command.type === 'maintenance') { + if (command.operation.kind === 'legacy-retirement-save') { + if ( + destinationMain !== legacyDestinationMain(workerData.filename) || + (command.operation.destination && + JSON.stringify(command.operation.destination) !== + JSON.stringify(legacyDestinationIdentity(workerData.filename))) + ) { + throw new Error('journal destination generation changed') + } + } return { type: 'maintenance', id, @@ -154,10 +171,17 @@ async function drain(): Promise { } }, writes.some( - (request) => request.command.type === 'write' && request.command.command.forkInstall + (request) => + request.command.type === 'maintenance' && + request.command.operation.kind === 'legacy-retirement-save' && + request.command.operation.durable ) - ? 'full' - : 'normal' + ? 'retirement' + : writes.some( + (request) => request.command.type === 'write' && request.command.command.forkInstall + ) + ? 'full' + : 'normal' ) forks!.acknowledge() // All acknowledged commits precede a fault notice; it never retracts a successful COMMIT. diff --git a/src/main/orcad/instance-lock/orcad-instance-lock-hold.ts b/src/main/orcad/instance-lock/orcad-instance-lock-hold.ts index 97574747cb5..bf3cb73a5f0 100644 --- a/src/main/orcad/instance-lock/orcad-instance-lock-hold.ts +++ b/src/main/orcad/instance-lock/orcad-instance-lock-hold.ts @@ -1,4 +1,5 @@ // Holding the lock means re-proving it on every heartbeat, not only at acquisition. +import { dirname, relative, isAbsolute } from 'node:path' import { statSync, unlinkSync, utimesSync } from 'node:fs' import { startOrcadLockHeartbeat } from './orcad-lock-clone-machine' import { readBoundedLockFile } from './orcad-instance-lock-reclaim' @@ -18,6 +19,22 @@ export type OrcadInstanceLock = { release(options?: { keepUnexpectedExit?: boolean }): void } +const ownedProfileLocks = new Map() + +export function ownsOrcadProfileDirectory(directory: string): boolean { + for (const [path, record] of ownedProfileLocks) { + const child = relative(dirname(path), directory) + if (child.startsWith('..') || isAbsolute(child)) { + continue + } + const current = parseOrcadInstanceLockRecord(readBoundedLockFile(path) ?? '') + if (current?.nonce === record.nonce && current.pid === process.pid) { + return true + } + } + return false +} + type OrcadLockBeat = 'held' | 'lost' | 'unproven' function lockIsAbsent(lockPath: string): boolean { @@ -89,6 +106,7 @@ export function holdOrcadInstanceLock( onExclusivityLost: () => void } ): OrcadInstanceLock { + ownedProfileLocks.set(lockPath, record) let released = false const stopHeartbeat = startOrcadLockHeartbeat(() => { if (beatOrcadInstanceLock(lockPath, record, publish) === 'lost') { @@ -105,6 +123,9 @@ export function holdOrcadInstanceLock( return } released = true + if (ownedProfileLocks.get(lockPath)?.nonce === record.nonce) { + ownedProfileLocks.delete(lockPath) + } stopHeartbeat() // Re-read ownership before unlinking so a successor’s live runtime cannot be unlocked. const current = parseOrcadInstanceLockRecord(readBoundedLockFile(lockPath) ?? '') diff --git a/src/shared/orcad-artifacts.ts b/src/shared/orcad-artifacts.ts index b561531a872..b66cd4c6650 100644 --- a/src/shared/orcad-artifacts.ts +++ b/src/shared/orcad-artifacts.ts @@ -75,6 +75,7 @@ export const ORCAD_NODE_PTY_JS_ARTIFACTS = [ export function orcadNodePtySlotFiles(target: string): string[] { if (isWindowsTarget(target)) { return [ + 'journal-file-lock.exe', 'conpty.node', 'conpty_console_list.node', 'conpty/conpty.dll', @@ -85,7 +86,9 @@ export function orcadNodePtySlotFiles(target: string): string[] { 'winpty-agent.exe' ] } - return target.startsWith('darwin-') ? ['pty.node', 'spawn-helper'] : ['pty.node'] + return target.startsWith('darwin-') + ? ['journal-file-lock', 'pty.node', 'spawn-helper'] + : ['journal-file-lock', 'pty.node'] } export function orcadNodePtyNativeArtifacts(target: string): string[] {