fix(native-chat): retire certified legacy journals after the grace period

This commit is contained in:
Brennan Benson
2026-11-13 05:12:29 -08:00
parent 4981f181b9
commit 9130ef8257
27 changed files with 1824 additions and 11 deletions
+30
View File
@@ -0,0 +1,30 @@
name: Journal file lock
on:
pull_request:
paths:
- 'native/journal-file-lock/**'
- 'config/scripts/native-build/build-journal-file-lock.mjs'
- 'src/main/native-chat/agent-session-journal/journal-legacy*'
- '.github/workflows/journal-file-lock.yml'
workflow_dispatch:
permissions:
contents: read
jobs:
lock:
strategy:
fail-fast: false
matrix:
os: [ubuntu-24.04, macos-15, windows-2025]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: ./.github/actions/install-node-dependencies
with:
native-runtime: node
node-version: '24'
- run: node config/scripts/native-build/build-journal-file-lock.mjs
- run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts
env:
ORCA_BACKGROUND_LAUNCH: '1'
+20
View File
@@ -166,6 +166,11 @@ const emojiShortcodeDatasetResource = {
to: 'node_modules/emojibase-data/en/shortcodes/emojibase.json'
}
const commonExtraResources = [
{
from: 'node_modules/node-pty/build/Release',
to: 'bin',
filter: ['journal-file-lock', 'journal-file-lock.exe']
},
relayExtraResource,
orcadTemplateExtraResource,
orcadTemplateNodeModulesExtraResource,
@@ -408,6 +413,16 @@ module.exports = {
if (context.electronPlatformName === 'linux') {
ensureWaylandClipboard(context.arch)
}
return import('./scripts/native-build/build-journal-file-lock.mjs').then(
({ buildJournalFileLock }) => {
const lockName =
context.electronPlatformName === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
buildJournalFileLock(
join(__dirname, '..', 'node_modules', 'node-pty', 'build', 'Release', lockName),
context.arch === 3 ? 'arm64' : 'x64'
)
}
)
},
afterPack: async (context) => {
const resourcesDir =
@@ -537,6 +552,11 @@ module.exports = {
})
chmodUnixCliLaunchers(resourcesDir, context.electronPlatformName)
if (context.electronPlatformName === 'darwin') {
await signMacStandaloneHelper(
join(resourcesDir, 'bin', 'journal-file-lock'),
'journal file lock',
context.packager
)
await signMacComputerUseHelper(join(resourcesDir, 'Orca Computer Use.app'), context.packager)
await signMacStandaloneHelper(
join(resourcesDir, '..', 'MacOS', 'orca-notification-status'),
@@ -0,0 +1,66 @@
import { mkdirSync } from 'node:fs'
import { dirname, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import {
initializeWindowsNativeToolchain,
mergeWindowsCompilerEnvironment
} from './windows/initialize-windows-native-toolchain.mjs'
import { runProcessSync } from '@orca/process-host'
export function buildJournalFileLock(output, arch = process.arch) {
const root = resolve(import.meta.dirname, '../../..')
mkdirSync(dirname(output), { recursive: true })
const source = resolve(root, 'native/journal-file-lock/main.c')
const windows = process.platform === 'win32'
const args = windows
? [
'/nologo',
'/O2',
'/W4',
'/WX',
'/DUNICODE',
'/D_UNICODE',
source,
`/Fe:${output}`,
`/Fo:${output}.obj`
]
: [
'-O2',
'-Wall',
'-Wextra',
'-Werror',
...(process.platform === 'darwin' ? ['-arch', arch === 'x64' ? 'x86_64' : arch] : []),
source,
'-o',
output
]
if (process.platform === 'linux' && arch !== process.arch) {
throw new Error('native lock needs a target-native Linux builder')
}
if (windows && arch !== process.arch) {
throw new Error('native lock needs a target-native Windows builder')
}
const compilerEnvironment = windows ? initializeWindowsNativeToolchain() : undefined
const result = runProcessSync({
program: windows ? 'cl.exe' : 'cc',
args,
cwd: root,
env: compilerEnvironment
? mergeWindowsCompilerEnvironment(process.env, compilerEnvironment)
: process.env,
timeoutMs: 120000
})
if (result.code !== 0) {
throw new Error(`journal lock build failed: ${result.stderr}${result.stdout}`)
}
return output
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
buildJournalFileLock(
resolve(
'native/journal-file-lock/.build',
process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
)
)
}
@@ -1,4 +1,5 @@
#!/usr/bin/env node
import { buildJournalFileLock } from '../../native-build/build-journal-file-lock.mjs'
// Build patched node-pty for current platform/arch/libc so deployment needs no compiler; compat slots also build COMPAT_SLOT_ADDONS.
// Require repo patch: GLIBC symbol pins plus no-as-needed retain libutil/libpthread and prevent #9902.
// Use pinned, hash-verified Node headers at N-API 8; glibc slots target 2.28, optional compat 2.17, musl stays separate.
@@ -254,6 +255,9 @@ async function build() {
? await compileCompatAddons({ slot, workDir: join(WORK_DIR, slot), nodeDir })
: []
buildJournalFileLock(
join(buildDir, process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock')
)
rmSync(slotDir, { recursive: true, force: true })
const files = {}
for (const [relative, source] of [
@@ -95,6 +95,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) {
const conptyRuntime = windowsConptyRuntimeDir(nodePtyDir, arch)
const upstream = join(nodePtyDir, 'prebuilds', `win32-${arch}`)
return [
['journal-file-lock.exe', join(buildDir, 'journal-file-lock.exe')],
['conpty.node', join(buildDir, 'conpty.node')],
['conpty_console_list.node', join(upstream, 'conpty_console_list.node')],
['conpty/conpty.dll', join(conptyRuntime, 'conpty.dll')],
@@ -104,6 +105,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) {
}
// macOS posix_spawns spawn-helper and builds it only there; Linux forks directly.
return [
['journal-file-lock', join(buildDir, 'journal-file-lock')],
['pty.node', join(buildDir, 'pty.node')],
...(platform === 'darwin' ? [['spawn-helper', join(buildDir, 'spawn-helper')]] : [])
]
@@ -23,6 +23,8 @@ export const WINDOWS_PREBUILD_LOCKED_PACKAGES = ['node-pty', 'node-gyp']
const WINDOWS_PREBUILD_PACKAGE_SCRIPTS = ['build:orcad-prebuilds', 'build:packages']
export const WINDOWS_PREBUILD_CACHE_INPUTS = [
'.github/actions/prepare-orcad-prebuilds/action.yml',
'native/journal-file-lock/main.c',
'config/scripts/native-build/build-journal-file-lock.mjs',
'config/patches/node-pty@1.1.0.patch',
'src/shared/node-runtime-pin.ts',
'config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs',
@@ -38,6 +38,7 @@ function fixture(arch = 'x64') {
mkdirSync(join(sourceDir, 'third_party/conpty/1', `win10-${arch}`), { recursive: true })
const files = {}
for (const [file, bytes] of [
['journal-file-lock.exe', peImage({ arch })],
[
'conpty.node',
Buffer.concat([peImage({ arch }), CYGWIN_BREAKAWAY_MARKER, NATIVE_HOST_JOB_MARKER])
@@ -51,7 +52,7 @@ function fixture(arch = 'x64') {
]) {
write(join(slotDir, file), bytes)
files[file] = sha256Of(join(slotDir, file))
if (file !== 'conpty.node' && !file.startsWith('conpty/')) {
if (file !== 'journal-file-lock.exe' && file !== 'conpty.node' && !file.startsWith('conpty/')) {
write(join(sourceDir, 'prebuilds', slot, file), bytes)
}
if (file.startsWith('conpty/')) {
+1
View File
@@ -0,0 +1 @@
.build/
+73
View File
@@ -0,0 +1,73 @@
/* Holds the standard SQLite main-file lock without opening or modifying SQLite. */
#ifdef _WIN32
#define _WIN32_WINNT 0x0602
#include <windows.h>
#include <stdio.h>
#include <wchar.h>
int wmain(int argc, wchar_t **argv) {
if (argc != 2) return 2;
HANDLE file = CreateFileW(argv[1], GENERIC_READ | DELETE, FILE_SHARE_DELETE,
NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL);
if (file == INVALID_HANDLE_VALUE) return 3;
BY_HANDLE_FILE_INFORMATION info;
wchar_t fs[32], volume[MAX_PATH], final[32768];
DWORD final_length = GetFinalPathNameByHandleW(file, final, 32768, FILE_NAME_NORMALIZED);
if (!GetFileInformationByHandle(file, &info) || info.nNumberOfLinks != 1 ||
(info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT)) ||
final_length == 0 || final_length >= 32768 ||
!GetVolumePathNameW(final, volume, MAX_PATH) || GetDriveTypeW(volume) != DRIVE_FIXED ||
!GetVolumeInformationByHandleW(file, NULL, 0, NULL, NULL, NULL, fs, 32) ||
wcscmp(fs, L"NTFS") != 0) {
CloseHandle(file); return 4;
}
HANDLE named = CreateFileW(argv[1], 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL);
BY_HANDLE_FILE_INFORMATION current;
int same = named != INVALID_HANDLE_VALUE && GetFileInformationByHandle(named, &current) &&
current.dwVolumeSerialNumber == info.dwVolumeSerialNumber &&
current.nFileIndexHigh == info.nFileIndexHigh && current.nFileIndexLow == info.nFileIndexLow;
if (named != INVALID_HANDLE_VALUE && !CloseHandle(named)) same = 0;
if (!same) { CloseHandle(file); return 4; }
puts("locked"); fflush(stdout);
while (getchar() != EOF) {}
return CloseHandle(file) ? 0 : 5;
}
#else
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#ifdef __APPLE__
#include <sys/mount.h>
#else
#include <sys/vfs.h>
#endif
int main(int argc, char **argv) {
if (argc != 2) return 2;
int fd = open(argv[1], O_RDWR | O_CLOEXEC | O_NOFOLLOW);
if (fd < 0) return 3;
struct stat info, named;
struct statfs volume;
int local = 0;
if (fstatfs(fd, &volume) == 0) {
#ifdef __APPLE__
local = strcmp(volume.f_fstypename, "apfs") == 0 || strcmp(volume.f_fstypename, "hfs") == 0;
#else
local = volume.f_type == 0xef53 || volume.f_type == 0x58465342 ||
volume.f_type == 0x9123683e || volume.f_type == 0x2fc12fc1;
#endif
}
struct flock lock = {0};
lock.l_type = F_WRLCK; lock.l_whence = SEEK_SET;
lock.l_start = 0x40000000; lock.l_len = 512;
if (!local || fstat(fd, &info) || !S_ISREG(info.st_mode) || info.st_nlink != 1 ||
fcntl(fd, F_SETLK, &lock) || lstat(argv[1], &named) ||
info.st_dev != named.st_dev || info.st_ino != named.st_ino) {
close(fd); return 4;
}
puts("locked"); fflush(stdout);
while (getchar() != EOF) {}
return close(fd) == 0 ? 0 : 5;
}
#endif
@@ -61,6 +61,7 @@ export class JournalSqlDatabase {
this.activity,
options.onMaintenanceError
)
this.maintenance.ownLegacySource(stateDirectory, this.readers)
}
static async open(
@@ -0,0 +1,88 @@
import { captureJournalLegacySourceReceipt } from './database/journal-legacy-source-receipt'
import { existsSync, mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { spawnProcess } from '@orca/process-host'
import { afterEach, expect, it } from 'vitest'
import { acquireJournalLegacyFileLock, journalLegacyFileLockPath } from './journal-legacy-file-lock'
import { openJournalDatabase } from './database/journal-database'
const cleanups: (() => void | Promise<void>)[] = []
afterEach(async () => {
for (const cleanup of cleanups.splice(0).toReversed()) {
await cleanup()
}
})
it.skipIf(!process.env.CI)(
'excludes a real SQLite process without altering source bytes, then releases on EOF',
async () => {
expect(existsSync(journalLegacyFileLockPath())).toBe(true)
const root = mkdtempSync(join(tmpdir(), 'orca-journal-file-lock-'))
cleanups.push(() => rmSync(root, { recursive: true, force: true }))
const filename = join(root, 'source.db')
const { db } = openJournalDatabase(filename)
db.exec(
"CREATE TABLE fixture_history(value TEXT); INSERT INTO fixture_history VALUES('history never opened')"
)
db.close()
const script = join(root, 'reader.cjs')
writeFileSync(
script,
"const{DatabaseSync}=require('node:sqlite');const db=new DatabaseSync(process.argv[2], {readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('ready');process.stdin.resume();process.stdin.on('end',()=>{db.close()});"
)
const reader = spawnProcess({
program: process.execPath,
args: [script, filename],
stdio: 'pipe'
})
const exited = new Promise<void>((resolve, reject) => {
reader.once('error', reject)
reader.once('exit', () => resolve())
})
cleanups.push(async () => {
reader.stdin.end()
await exited
})
await new Promise<void>((resolve, reject) => {
reader.stdout.once('data', () => resolve())
reader.once('error', reject)
})
const signal = new AbortController()
await expect(acquireJournalLegacyFileLock(filename, signal.signal)).rejects.toThrow()
reader.stdin.end()
await exited
const before = readFileSync(filename)
const identity = captureJournalLegacySourceReceipt(filename)
const lock = await acquireJournalLegacyFileLock(filename, signal.signal)
try {
expect(lock.held()).toBe(true)
expect(captureJournalLegacySourceReceipt(filename)).toEqual(identity)
const contender = spawnProcess({
program: process.execPath,
args: [
'-e',
"const{DatabaseSync}=require('node:sqlite');let db;try{db=new DatabaseSync(process.argv[1],{readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('opened')}catch{process.stdout.write('blocked')}finally{db?.close()}",
filename
],
stdio: 'pipe'
})
contender.stdin.end()
const output = await new Promise<string>((resolve, reject) => {
let result = ''
contender.stdout.on('data', (chunk) => {
result += chunk.toString()
})
contender.stderr.resume()
contender.once('error', reject)
contender.once('exit', () => resolve(result))
})
expect(output).toBe('blocked')
} finally {
await lock.close()
}
expect(readFileSync(filename)).toEqual(before)
const second = await acquireJournalLegacyFileLock(filename, signal.signal)
await second.close()
}
)
@@ -0,0 +1,77 @@
import { spawnProcess } from '@orca/process-host'
import { createRequire } from 'node:module'
import { existsSync } from 'node:fs'
import { dirname, join, resolve } from 'node:path'
import { currentWorkerEntryLayout } from '../../worker-threads/worker-thread-entry-path'
export function journalLegacyFileLockPath(): string {
const name = process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
const layout = currentWorkerEntryLayout(__dirname)
if (layout.isPackaged && layout.resourcesPath) {
return join(layout.resourcesPath, 'bin', name)
}
const require = createRequire(__filename)
const staged = join(dirname(require.resolve('node-pty/package.json')), 'build', 'Release', name)
return existsSync(staged) ? staged : resolve('native/journal-file-lock/.build', name)
}
/** The helper never opens SQLite; closing stdin releases its single non-writing source handle. */
export async function acquireJournalLegacyFileLock(filename: string, signal: AbortSignal) {
signal.throwIfAborted()
const child = spawnProcess({
program: journalLegacyFileLockPath(),
args: [filename],
stdio: 'pipe'
})
child.stdin.on('error', () => {})
child.stderr.resume()
let exited = false
const done = new Promise<void>((resolve, reject) => {
child.once('error', reject)
child.once('exit', (code) => {
exited = true
if (code === 0) {
resolve()
} else {
reject(new Error('legacy journal lock is unavailable'))
}
})
})
void done.catch(() => {})
const release = () => child.stdin.end()
signal.addEventListener('abort', release, { once: true })
if (signal.aborted) {
release()
}
try {
await new Promise<void>((resolve, reject) => {
let output = ''
child.stdout.on('data', (chunk: Buffer) => {
output += chunk.toString()
if (output === 'locked\n' || output === 'locked\r\n') {
resolve()
} else if (output.length > 64) {
reject(new Error('invalid legacy lock acknowledgement'))
}
})
child.once('error', reject)
child.once('exit', () => reject(new Error('legacy journal lock was refused')))
})
} catch (error) {
release()
await done.catch(() => {})
signal.removeEventListener('abort', release)
throw error
}
return {
held: () => !exited && !signal.aborted,
close: async () => {
release()
try {
await done
} finally {
signal.removeEventListener('abort', release)
}
}
}
}
@@ -0,0 +1,137 @@
import { lstat, open, stat, unlink } from 'node:fs/promises'
import { lstatSync } from 'node:fs'
import { dirname } from 'node:path'
import { journalDomainDatabasePath } from './database/journal-domain-database'
import {
journalLegacyFileIdentity,
type JournalLegacyFileIdentity
} from './database/journal-legacy-source-receipt'
export const LEGACY_COMPANION_SUFFIXES = ['-wal', '-shm'] as const
export type LegacyCompanionManifest = Partial<
Record<(typeof LEGACY_COMPANION_SUFFIXES)[number], JournalLegacyFileIdentity>
>
export type LegacyDestinationIdentity = { main: string; wal: string }
export function legacyDestinationMain(filename: string): string {
const file = lstatSync(filename, { bigint: true })
if (!file.isFile() || file.nlink !== 1n || file.ino === 0n) {
throw new Error('invalid journal destination identity')
}
return `${file.dev}:${file.ino}`
}
export function legacyDestinationIdentity(filename: string): LegacyDestinationIdentity {
return { main: legacyDestinationMain(filename), wal: legacyDestinationMain(`${filename}-wal`) }
}
export async function legacyFilePresent(filename: string): Promise<boolean> {
try {
await lstat(filename)
return true
} catch (error) {
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
return false
}
throw error
}
}
export async function syncLegacyDirectory(directory: string): Promise<void> {
if (process.platform === 'win32') {
return
}
// Ancestors are synced bottom-up so a newly created profile directory is covered too.
for (;;) {
const handle = await open(directory, 'r')
try {
await handle.sync()
} finally {
await handle.close()
}
const parent = dirname(directory)
if (parent === directory) {
return
}
directory = parent
}
}
export async function syncLegacyDestination(directory: string, source: string): Promise<void> {
const domain = journalDomainDatabasePath(directory)
const sourceFile = await stat(source, { bigint: true })
for (const name of [domain, `${domain}-wal`]) {
const file = await lstat(name, { bigint: true })
if (
!file.isFile() ||
file.dev !== sourceFile.dev ||
file.ino === sourceFile.ino ||
file.nlink !== 1n
) {
throw new Error('journal destination volume or identity changed')
}
}
if (process.platform === 'win32') {
const sourceDevice = sourceFile.dev
for (const name of [domain, `${domain}-wal`]) {
const handle = await open(name, 'r+')
try {
if ((await handle.stat({ bigint: true })).dev !== sourceDevice) {
throw new Error('journal destination volume changed')
}
await handle.sync()
} finally {
await handle.close()
}
}
}
await syncLegacyDirectory(directory)
}
export async function removeLegacyCompanions(
directory: string,
filename: string,
manifest: LegacyCompanionManifest | undefined,
boundary: () => Promise<boolean>,
ownsProfile: () => boolean,
signal: AbortSignal
): Promise<boolean> {
if (!manifest || !ownsProfile() || signal.aborted) {
return false
}
for (const suffix of LEGACY_COMPANION_SUFFIXES) {
if (
!(await boundary()) ||
!ownsProfile() ||
(await legacyFilePresent(filename)) ||
(await legacyFilePresent(`${filename}-journal`))
) {
return false
}
const name = filename + suffix
if (!(await legacyFilePresent(name))) {
continue
}
const expected = manifest[suffix]
const current = await lstat(name)
if (
!current.isFile() ||
!expected ||
JSON.stringify(journalLegacyFileIdentity(name)) !== JSON.stringify(expected) ||
(suffix === '-wal' && expected.size !== '0')
) {
throw new Error('legacy companion changed; retaining it')
}
await unlink(name)
}
if (
(await legacyFilePresent(filename)) ||
(
await Promise.all(
LEGACY_COMPANION_SUFFIXES.map((suffix) => legacyFilePresent(filename + suffix))
)
).some(Boolean)
) {
return false
}
await syncLegacyDirectory(directory)
return true
}
@@ -0,0 +1,106 @@
import type { LegacyCompanionManifest } from './journal-legacy-retirement-files'
export const LEGACY_RETIREMENT_WEEK = 7 * 24 * 60 * 60 * 1000
export type Retirement = {
version: 1
proof: string
proofId: string
completedAt: number
process: string
credit: number
creditRevision: number
lastWall: number
phase: 'grace' | 'prepared' | 'complete'
manifest?: LegacyCompanionManifest
}
export function decodeRetirement(value: string | null): Retirement | null {
if (!value) {
return null
}
const data: Retirement = JSON.parse(value)
if (
data.version !== 1 ||
typeof data.proof !== 'string' ||
typeof data.proofId !== 'string' ||
typeof data.process !== 'string' ||
!['grace', 'prepared', 'complete'].includes(data.phase) ||
!Number.isFinite(data.credit) ||
data.credit < 0 ||
data.credit > LEGACY_RETIREMENT_WEEK ||
!Number.isSafeInteger(data.creditRevision) ||
data.creditRevision < 0
) {
throw new Error('invalid legacy retirement receipt')
}
return data
}
export function advanceLegacyRetirementClock(
state: Retirement,
now: number,
elapsed: number,
processToken: string
): Retirement {
if (
!Number.isFinite(state.completedAt) ||
!Number.isFinite(state.lastWall) ||
state.completedAt < 0 ||
state.lastWall < 0 ||
state.completedAt > now ||
state.lastWall > now
) {
return {
...state,
completedAt: now,
lastWall: now,
credit: 0,
process: processToken,
phase: 'grace'
}
}
return {
...state,
credit: Math.min(
LEGACY_RETIREMENT_WEEK,
state.credit + Math.max(0, Number.isFinite(elapsed) ? elapsed : 0)
),
lastWall: now
}
}
/** Absolute revisions discard ambiguous saved credit instead of counting one interval twice. */
export class JournalLegacyGraceClock {
private anchor?: bigint
private revision?: string
private pending = 0
observe(state: Retirement, now: number, processToken: string): Retirement {
const revision = `${state.proofId}:${state.creditRevision}`
if (this.revision !== revision) {
this.pending = 0
this.anchor = undefined
this.revision = revision
}
const mono = process.hrtime.bigint()
if (this.anchor !== undefined && mono >= this.anchor) {
this.pending += Number(mono - this.anchor) / 1e6
}
this.anchor = mono
return advanceLegacyRetirementClock(state, now, this.pending, processToken)
}
persisted(state: Retirement): void {
this.revision = `${state.proofId}:${state.creditRevision}`
this.pending = 0
this.anchor = process.hrtime.bigint()
}
pause(): void {
const mono = process.hrtime.bigint()
if (this.anchor !== undefined && mono >= this.anchor) {
this.pending += Number(mono - this.anchor) / 1e6
}
this.anchor = undefined
}
}
@@ -0,0 +1,453 @@
import {
mkdtempSync,
rmSync,
existsSync,
writeFileSync,
readFileSync,
statSync,
readdirSync,
unlinkSync,
statfsSync
} from 'node:fs'
import { join, basename } from 'node:path'
import { tmpdir } from 'node:os'
import { afterEach, expect, it, vi } from 'vitest'
import Database from '../../sqlite/sync-database'
import { openJournalDatabase } from './database/journal-database'
import {
initializeJournalDomainDatabase,
openJournalDomainDatabase
} from './database/journal-domain-database'
import { journalLegacyFileIdentity } from './database/journal-legacy-source-receipt'
import { JournalSqlDatabase } from './database/journal-sql-database'
import {
journalSqlWorkerPath,
journalSqlReaderPath
} from './journal-sql-worker-bundle.test-fixture'
import { message } from './journal-sql-message.test-fixture'
import { journalDatabasePath } from './database/journal-host-database'
import { JournalSqlBatch } from './journal-sql-batch'
import { JournalLegacyRetirement } from './journal-legacy-retirement'
import {
advanceLegacyRetirementClock,
JournalLegacyGraceClock
} from './journal-legacy-retirement-state'
import { readJournalLegacyProofPage } from './journal-legacy-source-proof'
import { applyJournalSqlMaintenance, type JournalSqlMaintenance } from './journal-sql-maintenance'
import { runJournalTransaction } from './database/journal-database'
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
import { journalDomainImportState } from './journal-domain-import-rows'
import { prepareJournalDomainImport } from './journal-domain-import'
import { journalSqlRetryRows } from './journal-sql-retry-rows.test-fixture'
import { insertJournalRow, publishJournalSessionEpoch } from './durable-rows/journal-row-table'
const cleanup: (() => void)[] = []
afterEach(() => {
vi.restoreAllMocks()
for (const close of cleanup.splice(0).toReversed()) {
close()
}
})
async function profile(seed?: (db: Database.Database) => void) {
const root = mkdtempSync(join(tmpdir(), 'orca-legacy-retirement-'))
cleanup.push(() => rmSync(root, { recursive: true, force: true }))
const filename = journalDatabasePath(root)
const { db: legacy } = openJournalDatabase(filename)
runJournalTransaction(legacy, () => {
publishJournalSessionEpoch(
legacy,
{ sessionId: 'unopened', workspaceId: 'folder-without-git' },
'epoch'
)
for (const row of journalSqlRetryRows(true)) {
insertJournalRow(legacy, 'unopened', row)
}
})
seed?.(legacy)
legacy.close()
const { db } = openJournalDomainDatabase(await initializeJournalDomainDatabase(root))
let closed = false
const closeDomain = () => {
if (!closed) {
db.close()
closed = true
}
}
cleanup.push(closeDomain)
let original: Database.Database | undefined
const source = () =>
(original ??= new Database(filename, { readonly: true, fileMustExist: true }))
cleanup.push(() => original?.close())
const batch = new JournalSqlBatch(db)
const writer = {
maintain: async (operation: JournalSqlMaintenance) => {
const result = await batch.run(
[operation],
(unit) => applyJournalSqlMaintenance(db, unit),
operation.kind === 'legacy-retirement-save' && operation.durable ? 'retirement' : 'normal'
)
const outcome = result.outcomes[0]!
if (!outcome.committed) {
throw outcome.error
}
return outcome.value
}
}
const readers = {
legacyProof: async (cursor) => readJournalLegacyProofPage(source(), cursor, db),
importState: async (id) => journalDomainImportState(db, id),
recycleSourceReaders: async () => {
original?.close()
original = undefined
return new AbortController()
},
releaseSourceReaders: () => {}
} satisfies Pick<
JournalSqlReaderPool,
'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders'
>
const conversions = {
ensure: async (id: string) => {
if (journalDomainImportState(db, id).state === 'ready') {
return 'epoch'
}
const imported = prepareJournalDomainImport(source(), id)
if (!imported) {
return null
}
for (const page of imported.pages) {
await writer.maintain({ kind: 'import-page', target: imported, page })
}
return String(await writer.maintain({ kind: 'import-finish', header: imported }))
}
}
const errors: unknown[] = []
const lock = async () => ({ held: () => true, close: async () => {} })
const retirement = new JournalLegacyRetirement(
root,
writer,
readers,
conversions,
async () => true,
(error) => errors.push(error),
() => true,
lock
)
const receiptState = () =>
JSON.parse(
String(
db
.prepare(
"SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'"
)
.get()!.value
)
)
const saveState = (state: unknown) =>
db
.prepare(
"UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'"
)
.run(JSON.stringify(state))
const arm = () => {
const row = db
.prepare(
"SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'"
)
.get()
const state = JSON.parse(String(row!.value))
state.credit = 604800000
state.process = 'prior-process'
db.prepare(
"UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'"
).run(JSON.stringify(state))
}
return {
root,
filename,
db,
retirement,
errors,
arm,
source,
receiptState,
saveState,
writer,
closeDomain,
recycle: readers.recycleSourceReaders
}
}
it('accounts for an unopened folder chat, removes only the certified file and keeps current history', async () => {
const test = await profile()
const allocated = () => ({
files: Object.fromEntries(
readdirSync(test.root).map((name) => {
const file = statSync(join(test.root, name))
return [name, { logical: Number(file.size), allocated: Number(file.blocks) * 512 }]
})
),
freeBytes: Number(statfsSync(test.root).bfree) * Number(statfsSync(test.root).bsize)
})
const before = allocated()
const proofStart = performance.now()
await test.retirement.run()
expect(test.errors).toEqual([])
expect(existsSync(test.filename)).toBe(true)
const proofMs = performance.now() - proofStart
const proof = test.receiptState()
test.saveState({ ...proof, credit: 604800000 })
await test.retirement.run()
expect(existsSync(test.filename)).toBe(true)
test.arm()
const deletionStart = performance.now()
await test.retirement.run()
expect(test.errors).toEqual([])
expect(existsSync(test.filename)).toBe(false)
expect(
test.db
.prepare(
"SELECT payload_json FROM journal_items WHERE session_id='unopened' AND payload_json LIKE '%compact%' LIMIT 1"
)
.get()
).toBeDefined()
await test.retirement.run()
expect(existsSync(test.filename)).toBe(false)
test.db.pragma('wal_checkpoint(TRUNCATE)')
test.closeDomain()
const after = allocated()
const total = (files: typeof before.files) =>
Object.values(files).reduce((sum, value) => sum + value.allocated, 0)
console.info('synthetic allocated bytes after checkpoint and handle closure', {
before,
after,
grossLegacyRemoved: before.files[basename(test.filename)]!.allocated,
netRemoved: total(before.files) - total(after.files),
proofMs,
retireAndCheckpointMs: performance.now() - deletionStart
})
const options = {
writerPath: journalSqlWorkerPath(),
readerPath: journalSqlReaderPath(),
onMaintenanceError: () => {}
}
const current = await JournalSqlDatabase.open(test.root, options)
try {
const { target } = await current.openSession({
sessionId: 'new-current-chat',
workspaceId: 'folder-without-git',
hostId: 'synthetic-host',
agent: 'codex',
providerHandle: null
})
const step = message('saved after legacy retirement')
await current.write({
target,
savedAt: 100,
operation: {
kind: 'steps',
steps: [
{
kind: 'resolved-item',
reservedBytes: 1024,
resolution: { kind: 'text-snapshot', identity: step.identity, body: step.body },
options: step.options
}
]
}
})
} finally {
await current.close()
}
const reopened = await JournalSqlDatabase.open(test.root, options)
try {
expect(
await reopened.readers.domainHistory({
sessionId: 'new-current-chat',
direction: 'after',
limit: 10
})
).toMatchObject({
page: { items: [{ body: { blocks: [{ text: 'saved after legacy retirement' }] } }] }
})
expect(existsSync(test.filename)).toBe(false)
} finally {
await reopened.close()
}
})
it('retains downgrade writes, a rollback journal and a nonempty WAL without changing their bytes', async () => {
for (const kind of ['append', 'journal', 'wal'] as const) {
const test = await profile()
await test.retirement.run()
test.arm()
if (kind === 'append') {
const old = new Database(test.filename)
old
.prepare('INSERT INTO journal_sessions VALUES(?,?,?)')
.run('old-new-chat', 'folder', 'new-epoch')
const savedItem = journalSqlRetryRows(true).find((row) => row.kind === 'item')!
insertJournalRow(old, 'unopened', {
...savedItem,
seq: journalSqlRetryRows(true).length + 1,
itemId: 'downgraded-text',
body: {
kind: 'message',
role: 'user',
blocks: [{ type: 'text', text: 'saved by the downgraded build' }]
}
})
insertJournalRow(old, 'old-new-chat', {
...journalSqlRetryRows(true)[0]!,
epoch: 'new-epoch',
seq: 1
})
old.close()
} else {
writeFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'owed recovery bytes')
}
const before = readFileSync(test.filename)
await test.retirement.run()
expect(existsSync(test.filename)).toBe(true)
expect(readFileSync(test.filename)).toEqual(before)
if (kind === 'append') {
expect(
test
.source()
.prepare("SELECT session_id FROM journal_sessions WHERE session_id='old-new-chat'")
.get()
).toBeDefined()
expect(
test
.source()
.prepare(
"SELECT row_json FROM journal_rows WHERE session_id='unopened' ORDER BY seq DESC LIMIT 1"
)
.get()!.row_json
).toContain('saved by the downgraded build')
} else {
expect(readFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'utf8')).toBe(
'owed recovery bytes'
)
}
}
})
it('does not earn grace from forward wall-clock steps and re-stamps a future completion', () => {
const state = {
version: 1 as const,
proof: 'proof',
proofId: 'id',
completedAt: 100,
process: 'prior',
credit: 0,
creditRevision: 0,
lastWall: 100,
phase: 'grace' as const
}
expect(advanceLegacyRetirementClock(state, 691200100, 3600000, 'next').credit).toBe(3600000)
expect(
advanceLegacyRetirementClock(
{ ...state, completedAt: 9000, credit: 604800000 },
1000,
10,
'next'
)
).toMatchObject({ completedAt: 1000, credit: 0, process: 'next' })
let mono = 0n
vi.spyOn(process.hrtime, 'bigint').mockImplementation(() => mono)
const clock = new JournalLegacyGraceClock()
clock.persisted(state)
mono = 3600000000000n
expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000)
clock.pause()
expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000)
const ambiguousCommit = { ...state, credit: 3600000, creditRevision: 1 }
mono = 7200000000000n
expect(clock.observe(ambiguousCommit, 691200100, 'next').credit).toBe(3600000)
})
it('retains queued-only text, recovery obligations and orphan history', async () => {
for (const kind of ['queue', 'repair', 'orphan', 'extra-profile-column'] as const) {
const test = await profile((legacy) => {
if (kind === 'queue') {
legacy.exec(
'DELETE FROM journal_rows; INSERT INTO queued_messages(session_id,message_id,position,body_json,fingerprint,created_at,host_instance,state,hold_reason) VALUES(\'unopened\',\'held\',1,\'{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}\',\'f\',1,\'h\',\'waiting\',\'manual\')'
)
} else if (kind === 'repair') {
legacy.exec("INSERT INTO journal_repairs VALUES('unopened','epoch',1,100)")
} else if (kind === 'extra-profile-column') {
legacy.exec(
"ALTER TABLE agent_session_records ADD COLUMN future_text TEXT; INSERT INTO agent_session_records VALUES('unopened','{}','saved future field')"
)
} else {
legacy.exec("INSERT INTO journal_rows VALUES('orphan','old',1,1,'orphan user text')")
}
})
if (kind === 'queue') {
const current = await JournalSqlDatabase.open(test.root, {
writerPath: journalSqlWorkerPath(),
readerPath: journalSqlReaderPath(),
onMaintenanceError: () => {}
})
try {
await current.openSession({
sessionId: 'unopened',
workspaceId: 'folder-without-git',
hostId: 'synthetic-host',
agent: 'codex',
providerHandle: null
})
} finally {
await current.close()
}
}
const before = readFileSync(test.filename)
await test.retirement.run()
expect(test.errors.length).toBeGreaterThan(0)
expect(readFileSync(test.filename)).toEqual(before)
if (kind === 'queue') {
expect(test.source().prepare('SELECT body_json FROM queued_messages').get()).toEqual({
body_json:
'{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}'
})
}
}
})
it('resumes only the prepared empty companions and preserves a recreated main', async () => {
for (const recreate of [false, true]) {
const test = await profile()
await test.retirement.run()
await test.recycle()
const shm = `${test.filename}-shm`
writeFileSync(shm, 'derived index')
const state = {
...test.receiptState(),
phase: 'prepared',
manifest: {
'-shm': journalLegacyFileIdentity(shm),
...(existsSync(`${test.filename}-wal`)
? { '-wal': journalLegacyFileIdentity(`${test.filename}-wal`) }
: {})
}
}
test.saveState(state)
unlinkSync(test.filename)
if (recreate) {
writeFileSync(test.filename, 'new old-build history')
}
await test.retirement.run()
if (recreate) {
expect(readFileSync(shm, 'utf8')).toBe('derived index')
}
if (recreate) {
expect(readFileSync(test.filename, 'utf8')).toBe('new old-build history')
} else {
expect(test.errors).toEqual([])
expect(existsSync(shm)).toBe(false)
}
}
})
@@ -0,0 +1,303 @@
import { randomUUID } from 'node:crypto'
import { lstat, unlink } from 'node:fs/promises'
import { ownsOrcadProfileDirectory } from '../../orcad/instance-lock/orcad-instance-lock-hold'
import { journalDatabasePath } from './database/journal-host-database'
import { journalDomainDatabasePath } from './database/journal-domain-database'
import {
LEGACY_COMPANION_SUFFIXES as SUFFIXES,
legacyFilePresent as filePresent,
syncLegacyDirectory,
removeLegacyCompanions,
syncLegacyDestination,
legacyDestinationIdentity,
type LegacyCompanionManifest as Manifest,
type LegacyDestinationIdentity
} from './journal-legacy-retirement-files'
import {
LEGACY_RETIREMENT_WEEK,
decodeRetirement,
JournalLegacyGraceClock,
type Retirement
} from './journal-legacy-retirement-state'
import {
captureJournalLegacySourceReceipt,
journalLegacyFileIdentity
} from './database/journal-legacy-source-receipt'
import { acquireJournalLegacyFileLock } from './journal-legacy-file-lock'
import type { JournalSqlOwner } from './journal-sql-owner'
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
import type { JournalSqlConversions } from './journal-sql-conversions'
import { completeJournalLegacyProof } from './journal-legacy-source-proof'
const WEEK = LEGACY_RETIREMENT_WEEK
const SAVE_INTERVAL = 6 * 60 * 60 * 1000
const PROCESS_TOKEN = randomUUID()
/** Existing maintenance owns proof, grace and one fixed unlink per yield; failures only retain data. */
export class JournalLegacyRetirement {
private readonly filename: string
private readonly clock = new JournalLegacyGraceClock()
private lastSaved = 0
private diagnosed?: string
private stopped = new AbortController()
constructor(
private readonly directory: string,
private readonly writer: Pick<JournalSqlOwner, 'maintain'>,
private readonly readers: Pick<
JournalSqlReaderPool,
'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders'
>,
private readonly conversions: Pick<JournalSqlConversions, 'ensure'>,
private readonly boundary: () => Promise<boolean>,
private readonly report: (error: unknown) => void,
private readonly ownsProfile = () => ownsOrcadProfileDirectory(directory),
private readonly lock = acquireJournalLegacyFileLock
) {
this.filename = journalDatabasePath(directory)
}
stop(): void {
if (!this.stopped.signal.aborted) {
this.clock.pause()
this.stopped.abort()
}
}
async run(): Promise<void> {
try {
await this.attempt()
} catch (error) {
const reason = error instanceof Error ? error.message : String(error)
if (this.diagnosed !== reason) {
this.diagnosed = reason
this.report(error)
}
}
}
async flushCredit(): Promise<void> {
if (!this.stopped.signal.aborted) {
return
}
try {
const saved = await this.savedState()
const state = decodeRetirement(saved.retirement)
if (
!state ||
state.phase !== 'grace' ||
state.proof !== saved.receipt ||
!this.matches(state.proof)
) {
return
}
const advanced = this.clock.observe(state, Date.now(), PROCESS_TOKEN)
if (advanced.completedAt === state.completedAt && advanced.credit > state.credit) {
await this.save(advanced, false)
}
} catch (error) {
this.report(error)
}
}
private async savedState() {
const saved = await this.writer.maintain({ kind: 'legacy-retirement-state' })
if (
!saved ||
typeof saved !== 'object' ||
!('kind' in saved) ||
saved.kind !== 'legacy-retirement-state'
) {
throw new Error('legacy retirement state is unavailable')
}
return saved
}
private async save(
state: Retirement,
durable: boolean,
destination?: LegacyDestinationIdentity
): Promise<void> {
state.creditRevision++
if (
(await this.writer.maintain({
kind: 'legacy-retirement-save',
value: JSON.stringify(state),
durable,
destination
})) !== true
) {
throw new Error('legacy retirement commit was not acknowledged')
}
this.lastSaved = Date.now()
this.clock.persisted(state)
}
private matches(proof: string): boolean {
return JSON.stringify(captureJournalLegacySourceReceipt(this.filename)) === proof
}
private prove(): Promise<void> {
return completeJournalLegacyProof(
this.readers,
this.conversions,
this.writer,
this.boundary,
this.stopped.signal
)
}
private async attempt(): Promise<void> {
if (this.stopped.signal.aborted) {
return
}
const saved = await this.savedState()
if (!saved.receipt) {
return
}
const receipt = JSON.parse(saved.receipt)
if (
receipt.version !== 1 ||
receipt.profileCopyInventoryVersion !== 1 ||
receipt.disposition !== 'copied-source' ||
receipt.wal?.kind !== 'empty'
) {
return
}
const proof = saved.receipt
let state = decodeRetirement(saved.retirement)
if (state && state.proof !== saved.receipt) {
throw new Error('legacy retirement provenance changed')
}
if (state?.phase === 'complete') {
if (!(await filePresent(this.filename))) {
return
}
state = { ...state, phase: 'prepared' }
}
if (state?.phase === 'prepared' && !(await filePresent(this.filename))) {
await this.removeCompanions(state)
return
}
if (!this.matches(proof)) {
throw new Error('legacy source changed; retaining the original')
}
const now = Date.now()
if (!state) {
await this.prove()
if (!this.matches(proof)) {
throw new Error('legacy source changed during proof')
}
state = {
version: 1,
proof: saved.receipt,
proofId: randomUUID(),
completedAt: now,
process: PROCESS_TOKEN,
credit: 0,
creditRevision: 0,
lastWall: now,
phase: 'grace'
}
await this.save(state, true)
return
}
const advanced = this.clock.observe(state, now, PROCESS_TOKEN)
if (advanced.process !== state.process || advanced.completedAt !== state.completedAt) {
await this.prove()
if (!this.matches(proof)) {
return
}
await this.save(advanced, true)
return
}
state = advanced
if (state.credit < WEEK || state.process === PROCESS_TOKEN) {
if (now - this.lastSaved >= SAVE_INTERVAL) {
await this.save(state, false)
}
return
}
await this.prove()
if (!this.ownsProfile() || !this.matches(proof)) {
return
}
const recycled = await this.readers.recycleSourceReaders()
if (!recycled) {
return
}
const signal = AbortSignal.any([recycled.signal, this.stopped.signal])
let lease: Awaited<ReturnType<typeof acquireJournalLegacyFileLock>> | undefined
try {
lease = await this.lock(this.filename, signal)
const validate = () => {
if (!lease?.held() || signal.aborted || !this.ownsProfile() || !this.matches(proof)) {
throw new Error('legacy retirement lost exclusive ownership')
}
}
validate()
if (await filePresent(`${this.filename}-journal`)) {
throw new Error('legacy rollback journal retained')
}
const manifest: Manifest = {}
for (const suffix of SUFFIXES) {
if (await filePresent(this.filename + suffix)) {
const file = await lstat(this.filename + suffix)
if (!file.isFile()) {
throw new Error('legacy companion is not a regular file')
}
manifest[suffix] = journalLegacyFileIdentity(this.filename + suffix)
}
}
if (manifest['-wal'] && manifest['-wal'].size !== '0') {
throw new Error('legacy WAL still contains data')
}
const domain = journalDomainDatabasePath(this.directory)
const destination = legacyDestinationIdentity(domain)
await syncLegacyDestination(this.directory, this.filename)
if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) {
throw new Error('journal destination changed during sync')
}
state = { ...state, phase: 'prepared', manifest }
await this.save(state, true, destination)
validate()
if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) {
throw new Error('journal destination changed after prepare')
}
if (process.platform === 'win32') {
await lease.close()
lease = undefined
}
if (!this.ownsProfile() || signal.aborted || !this.matches(proof)) {
return
}
await unlink(this.filename)
await syncLegacyDirectory(this.directory)
await lease?.close()
lease = undefined
this.readers.releaseSourceReaders(recycled)
await this.removeCompanions(state)
} finally {
try {
await lease?.close()
} finally {
this.readers.releaseSourceReaders(recycled)
}
}
}
private async removeCompanions(state: Retirement): Promise<void> {
if (
await removeLegacyCompanions(
this.directory,
this.filename,
state.manifest,
this.boundary,
this.ownsProfile,
this.stopped.signal
)
) {
await this.save({ ...state, phase: 'complete' }, true)
}
}
}
@@ -0,0 +1,252 @@
import { readStoredQueuedMessageRow } from './queued-messages/queued-message-stored-row'
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
import type { JournalSqlConversions } from './journal-sql-conversions'
import type { JournalSqlOwner } from './journal-sql-owner'
import type Database from '../../sqlite/sync-database'
const PROFILE_TABLES = new Set([
'agent_session_records',
'agent_session_operations',
'agent_session_retired_claim_keys',
'agent_session_tabs',
'agent_session_store_meta',
'agent_session_command_receipts',
'agent_session_attachment_claims',
'agent_session_attachment_sweeps',
'pr_chat_actions',
'pr_check_events',
'pr_event_deliveries',
'pr_pending_heads',
'pr_pending_actions',
'pr_check_contexts',
'sqlite_sequence'
])
const CONTENT_TABLES = [
'journal_rows',
'journal_outcome_generations',
'journal_outcome_epochs',
'journal_outcomes',
'queued_messages'
] as const
const SOURCE_COLUMNS: Readonly<Record<string, string>> = {
journal_sessions: 'session_id workspace_id epoch',
journal_rows: 'session_id epoch seq ts row_json',
journal_outcome_generations: 'generation session_id epoch',
journal_outcome_epochs: 'session_id epoch generation format through_seq prefix_seq',
journal_outcomes: 'generation seq field entry_key format ordinal value_json deleted',
queued_messages:
'session_id message_id position body_json fingerprint created_at host_instance state hold_reason returned_reason returned_rejection settled_at settled_by_op consumed_as carried_from queued_epoch queued_sequence options_json'
}
const RECOVERY_TABLES = ['journal_repairs', 'journal_imports', 'journal_set_aside']
export type LegacyProofCursor = { table: number; after: string | null }
export type LegacySourceSession = {
sessionId: string
workspaceId: string
epoch: string
empty: boolean
}
export type LegacyProofPage = { next: LegacyProofCursor | null; sessions: LegacySourceSession[] }
/** Every saved content row must belong to the current catalog before any source file can retire. */
export function readJournalLegacyProofPage(
db: Database.Database,
cursor: LegacyProofCursor,
destination: Database.Database
): LegacyProofPage {
const tables = new Set(
db
.prepare("SELECT name FROM sqlite_schema WHERE type='table'")
.all()
.map((row) => String(row.name))
)
if (cursor.table === 0 && cursor.after === null) {
const version = Number(db.pragma('user_version', { simple: true }))
if (version !== 3 && version !== 4) {
throw new Error('unsupported legacy source inventory')
}
for (const table of tables) {
const names =
PROFILE_TABLES.has(table) && table !== 'sqlite_sequence'
? destination
.prepare(`PRAGMA table_info(${table})`)
.all()
.map((column) => column.name)
: SOURCE_COLUMNS[table]?.split(' ')
if (names) {
const columns = new Set(names)
if (
db
.prepare(`PRAGMA table_info(${table})`)
.all()
.some((column) => !columns.has(column.name))
) {
throw new Error('legacy source has unaccounted columns')
}
}
if (
PROFILE_TABLES.has(table) ||
CONTENT_TABLES.includes(table as (typeof CONTENT_TABLES)[number]) ||
table === 'journal_sessions'
) {
continue
}
const quoted = `"${table.replaceAll('"', '""')}"`
if (db.prepare(`SELECT 1 FROM ${quoted} LIMIT 1`).get()) {
throw new Error('legacy source has unaccounted recovery data')
}
if (!RECOVERY_TABLES.includes(table)) {
throw new Error('legacy source has an unknown table')
}
}
}
if (cursor.table < CONTENT_TABLES.length) {
const table = CONTENT_TABLES[cursor.table]!
if (!tables.has(table)) {
return { next: { table: cursor.table + 1, after: null }, sessions: [] }
}
const columns =
table === 'journal_outcomes'
? 'generation'
: table === 'queued_messages'
? `session_id,${
db
.prepare('PRAGMA table_info(queued_messages)')
.all()
.some((column) => column.name === 'queued_epoch')
? 'queued_epoch'
: 'NULL'
} AS epoch`
: table === 'journal_rows'
? 'session_id,epoch'
: 'session_id,epoch,generation'
const statement = db.prepare(
`SELECT rowid AS rowid,${columns} FROM ${table} ${cursor.after === null ? '' : 'WHERE rowid>?'} ORDER BY rowid LIMIT 128`
)
statement.setReadBigInts(true)
const rows = statement.all(...(cursor.after === null ? [] : [BigInt(cursor.after)]))
for (const row of rows) {
if (
table === 'queued_messages' &&
!readStoredQueuedMessageRow(
db.prepare('SELECT * FROM queued_messages WHERE rowid=?').get(row.rowid!)
)
) {
throw new Error('legacy source has unreadable queued text')
}
const session =
table === 'journal_outcomes'
? db
.prepare(
'SELECT session_id,epoch FROM journal_outcome_generations WHERE generation=?'
)
.get(row.generation!)
: row
if (
!session ||
!db
.prepare(
`SELECT 1 FROM journal_sessions WHERE session_id=?${
table === 'queued_messages' ? '' : ' AND epoch=?'
}`
)
.get(
session.session_id!,
...(table === 'queued_messages' && session.epoch == null ? [] : [session.epoch!])
)
) {
throw new Error('legacy source has orphan content')
}
if (table.startsWith('journal_outcome')) {
const generation = row.generation
if (
!db
.prepare(
'SELECT 1 FROM journal_outcome_epochs WHERE session_id=? AND epoch=? AND generation=?'
)
.get(session.session_id!, session.epoch!, generation!)
) {
throw new Error('legacy source has an orphan outcome generation')
}
}
}
return {
next: rows.length
? { table: cursor.table, after: String(rows.at(-1)!.rowid) }
: { table: cursor.table + 1, after: null },
sessions: []
}
}
const rows = db
.prepare(
`SELECT session_id,workspace_id,epoch FROM journal_sessions ${cursor.after === null ? '' : 'WHERE session_id>?'} ORDER BY session_id LIMIT 32`
)
.all(...(cursor.after === null ? [] : [cursor.after]))
const sessions = rows.map((row) => {
const id = String(row.session_id)
const hasRows = !!db.prepare('SELECT 1 FROM journal_rows WHERE session_id=? LIMIT 1').get(id)
const hasQueue =
tables.has('queued_messages') &&
!!db.prepare('SELECT 1 FROM queued_messages WHERE session_id=? LIMIT 1').get(id)
if (!hasRows && hasQueue) {
throw new Error('legacy queued-only history still requires its source')
}
const empty =
!hasRows &&
!hasQueue &&
(!tables.has('journal_outcome_epochs') ||
!db.prepare('SELECT 1 FROM journal_outcome_epochs WHERE session_id=? LIMIT 1').get(id))
return { sessionId: id, workspaceId: String(row.workspace_id), epoch: String(row.epoch), empty }
})
return {
next: rows.length ? { table: cursor.table, after: String(rows.at(-1)!.session_id) } : null,
sessions
}
}
export async function completeJournalLegacyProof(
readers: Pick<JournalSqlReaderPool, 'legacyProof' | 'importState'>,
conversions: Pick<JournalSqlConversions, 'ensure'>,
writer: Pick<JournalSqlOwner, 'maintain'>,
boundary: () => Promise<boolean>,
signal: AbortSignal
): Promise<void> {
let cursor: LegacyProofCursor | null = { table: 0, after: null }
while (cursor && !signal.aborted) {
if (!(await boundary())) {
throw new Error('legacy proof was interrupted')
}
const page = await readers.legacyProof(cursor)
for (const source of page.sessions) {
await conversions.ensure(source.sessionId, false)
if ((await readers.importState(source.sessionId)).state !== 'ready' && source.empty) {
await writer.maintain({
kind: 'import-finish',
header: {
sessionId: source.sessionId,
workspaceId: source.workspaceId,
epoch: source.epoch,
head: {
lastSequence: 0,
highestFence: 0,
contextBoundary: 0,
lastActivityAt: 0,
derivedTurnItemId: null,
historyOrigin: null,
queueFailureBoundary: 0,
queuePauseMarks: { latestStop: null, resumedSequence: 0, reopenedSequence: 0 }
}
}
})
}
if ((await readers.importState(source.sessionId)).state !== 'ready') {
throw new Error('legacy source still has unconverted or queued history')
}
}
cursor = page.next
}
if (signal.aborted) {
throw new Error('legacy proof was interrupted')
}
}
@@ -1,3 +1,4 @@
import { JournalLegacyRetirement } from './journal-legacy-retirement'
import { setImmediate, setTimeout as delay } from 'node:timers/promises'
import type { JournalSqlOwner } from './journal-sql-owner'
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
@@ -30,6 +31,19 @@ export class JournalSqlBackgroundMaintenance {
private readonly onError: (error: unknown, sessionId?: string) => void
) {}
private legacyRetirement?: JournalLegacyRetirement
ownLegacySource(directory: string, readers: JournalSqlReaderPool): void {
this.legacyRetirement = new JournalLegacyRetirement(
directory,
this.writer,
readers,
this.conversions,
() => this.boundary(),
this.onError
)
}
start(): Promise<void> {
if (this.stopped.signal.aborted) {
return Promise.resolve()
@@ -75,13 +89,15 @@ export class JournalSqlBackgroundMaintenance {
}
stop(): void {
this.legacyRetirement?.stop()
this.stopped.abort()
clearTimeout(this.timer)
this.timer = undefined
}
settled(): Promise<void> {
return this.running ?? Promise.resolve()
async settled(): Promise<void> {
await this.running
await this.legacyRetirement?.flushCredit()
}
private async boundary(): Promise<boolean> {
@@ -102,6 +118,7 @@ export class JournalSqlBackgroundMaintenance {
private async run(): Promise<void> {
while (!this.stopped.signal.aborted && (this.discover || this.sweep || this.periodic)) {
await this.runOrdinaryWork()
await this.legacyRetirement?.run()
if (this.discover) {
this.discover = false
try {
@@ -37,7 +37,7 @@ export class JournalSqlBatch {
async run<Command, Value>(
commands: readonly Command[],
apply: (command: Command) => Value | Promise<Value>,
durability: 'normal' | 'full' = 'normal'
durability: 'normal' | 'full' | 'retirement' = 'normal'
): Promise<JournalSqlBatchResult<Value>> {
if (this.failure) {
throw new JournalSqlBatchError('not-dispatched', this.failure)
@@ -47,13 +47,24 @@ export class JournalSqlBatch {
return { outcomes }
}
const drain = commands.slice()
const pragmas =
durability === 'retirement'
? ['synchronous', 'busy_timeout', 'fullfsync', 'checkpoint_fullfsync'].map(
(name) => [name, this.db.pragma(name, { simple: true })] as const
)
: []
let began = false
let committing = false
let failed: JournalSqlBatchError | undefined
try {
await this.checkpoint?.beforeWrite()
if (durability === 'full') {
if (durability !== 'normal') {
this.db.pragma('synchronous=FULL')
if (durability === 'retirement') {
this.db.pragma('busy_timeout=0')
this.db.pragma('fullfsync=ON')
this.db.pragma('checkpoint_fullfsync=ON')
}
}
this.db.exec('BEGIN IMMEDIATE')
began = true
@@ -89,9 +100,15 @@ export class JournalSqlBatch {
this.failure = failed
}
} finally {
if (durability === 'full' && !this.db.isTransaction) {
if (durability !== 'normal' && !this.db.isTransaction) {
try {
this.db.pragma('synchronous=NORMAL')
if (durability === 'retirement') {
for (const [name, value] of pragmas) {
this.db.pragma(`${name}=${Number(value)}`)
}
} else {
this.db.pragma('synchronous=NORMAL')
}
} catch (cause) {
failed ??= new JournalSqlBatchError('indeterminate', cause)
this.failure = failed
@@ -1,3 +1,4 @@
import type { LegacyDestinationIdentity } from './journal-legacy-retirement-files'
import { rekeyWslDriveUncChats } from '../../wsl/filesystem/wsl-drive-unc-chat-rekey'
import type { JournalForkStageOperation, JournalForkStageResult } from './journal-fork-stages'
import { saveJournalDomainOutlineCache, type JournalOutlineCache } from './journal-domain-outline'
@@ -26,6 +27,13 @@ import { installAgentSessionStoreRows } from '../../runtime/agent-session/record
import type { JournalSqlStoredRecords } from './journal-sql-database-read'
export type JournalSqlMaintenance =
| { kind: 'legacy-retirement-state' }
| {
kind: 'legacy-retirement-save'
value: string
durable: boolean
destination?: LegacyDestinationIdentity
}
| { kind: 'records-install' }
| { kind: 'wsl-workspace-rekey'; worktreeIds: readonly (readonly [string, string])[] }
| JournalForkStageOperation
@@ -44,6 +52,7 @@ export type JournalSqlMaintenance =
export type JournalSqlProjectionState = JournalDomainImportState
export type JournalSqlMaintenanceResult =
| { kind: 'legacy-retirement-state'; receipt: string | null; retirement: string | null }
| { kind: 'records'; records: JournalSqlStoredRecords }
| JournalForkStageResult
| JournalSqlProjectionState
@@ -59,6 +68,25 @@ export function applyJournalSqlMaintenance(
operation: JournalSqlMaintenance
): JournalSqlMaintenanceResult {
switch (operation.kind) {
case 'legacy-retirement-state': {
const value = (key: string) =>
db
.prepare('SELECT value FROM agent_session_store_meta WHERE key=?')
.get(key)
?.value?.toString() ?? null
return {
kind: 'legacy-retirement-state',
receipt: value('journal.legacy-source-copy.v1'),
retirement: value('journal.legacy-retirement.v1')
}
}
case 'legacy-retirement-save':
db.prepare('INSERT OR REPLACE INTO agent_session_store_meta(key,value) VALUES(?,?)').run(
'journal.legacy-retirement.v1',
operation.value
)
return true
case 'wsl-workspace-rekey':
return rekeyWslDriveUncChats(db, operation.worktreeIds)
case 'records-install': {
@@ -207,3 +207,21 @@ it('correlates interleaved replies and resumes current bodies after writer closu
})
expect(reset.page.items).toHaveLength(1)
})
it('serves domain reads while source recycling yields to a new chat', async () => {
const { filename, root } = await fixture()
const pool = new JournalSqlReaderPool(
filename,
join(root, 'absent-legacy.db'),
journalSqlReaderPath()
)
readers.push(pool)
await pool.importState('session')
const lease = await pool.recycleSourceReaders()
expect(lease).not.toBeNull()
const conversion = pool.prepareConversion('new-current-chat')
expect(await pool.importState('session')).toMatchObject({ state: 'ready' })
pool.releaseSourceReaders(lease!)
expect(await conversion).toBeNull()
expect(await pool.importState('session')).toMatchObject({ state: 'ready' })
})
@@ -1,3 +1,4 @@
import { readJournalLegacyProofPage } from './journal-legacy-source-proof'
import { JournalForkPreparation } from './journal-fork-preparation'
import { parentPort, workerData } from 'node:worker_threads'
import { existsSync } from 'node:fs'
@@ -83,6 +84,7 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => {
case 'database':
reply({ type: 'database', id, result: readJournalSqlDatabase(db, operation.operation) })
break
case 'legacy-proof':
case 'conversion-prepare': {
// Each frozen-source read is its own short statement; no snapshot spans the restore.
if (!original && existsSync(workerData.originalFilename)) {
@@ -92,6 +94,17 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => {
timeout: 5000
})
}
if (operation.kind === 'legacy-proof') {
if (!original) {
throw new Error('legacy source disappeared during proof')
}
reply({
type: 'legacy-proof',
id,
page: readJournalLegacyProofPage(original, operation.cursor, db)
})
break
}
const prepared = original ? prepareJournalDomainImport(original, operation.sessionId) : null
if (prepared) {
const { pages, ...header } = prepared
@@ -1,3 +1,4 @@
import type { LegacyProofCursor } from './journal-legacy-source-proof'
import type { JournalSqlForkReader, JournalSqlConversionReader } from './journal-sql-reader-slot'
import { ReaderSlot } from './journal-sql-reader-slot'
import type { JournalForkPreparationInput } from './journal-fork-preparation'
@@ -22,6 +23,8 @@ export class JournalSqlReaderPool {
private closed = false
private closing?: Promise<void>
private conversionSlot?: ReaderSlot
private sourceLease?: AbortController
private sourceRelease?: PromiseWithResolvers<void>
private forkSlot?: ReaderSlot
private forkRelease?: PromiseWithResolvers<void>
@@ -99,12 +102,61 @@ export class JournalSqlReaderPool {
}
async prepareConversion(sessionId: string): Promise<JournalSqlConversionReader | null> {
await this.releaseSourceLease()
if (!this.conversionSlot || this.conversionSlot.failed) {
this.conversionSlot = this.slot()
}
return this.conversionSlot.prepareConversion(sessionId)
}
async legacyProof(cursor: LegacyProofCursor) {
await this.releaseSourceLease()
this.conversionSlot ??= this.slot()
const response = await this.conversionSlot.request({ kind: 'legacy-proof', cursor })
if (response.type !== 'legacy-proof') {
throw new Error('legacy proof reader returned no page')
}
return response.page
}
async recycleSourceReaders(): Promise<AbortController | null> {
if (
this.closed ||
this.sourceLease ||
this.forkSlot ||
this.slots.some((slot) => slot.pendingCount !== 0)
) {
return null
}
const lease = new AbortController()
this.sourceLease = lease
this.sourceRelease = Promise.withResolvers<void>()
const retired = this.slots.splice(0)
this.conversionSlot = undefined
try {
await Promise.all([...this.retiring, ...retired.map((slot) => slot.close())])
return lease
} catch (error) {
this.releaseSourceReaders(lease)
throw error
}
}
releaseSourceReaders(lease: AbortController): void {
if (this.sourceLease === lease) {
this.sourceLease = undefined
this.sourceRelease?.resolve()
this.sourceRelease = undefined
}
}
private async releaseSourceLease(): Promise<void> {
if (this.sourceLease) {
this.sourceLease.abort()
await this.sourceRelease?.promise
}
}
async importState(sessionId: string) {
const response = await this.slot().request({ kind: 'import-state', sessionId })
if (response.type !== 'import-state') {
@@ -159,6 +211,7 @@ export class JournalSqlReaderPool {
}
close(): Promise<void> {
this.sourceLease?.abort()
this.closed = true
this.forkRelease?.resolve()
return (this.closing ??= Promise.all([
@@ -1,3 +1,4 @@
import type { LegacyProofCursor, LegacyProofPage } from './journal-legacy-source-proof'
import type {
JournalForkPreparationInput,
JournalForkPreparationResult
@@ -14,6 +15,7 @@ import type { JournalDomainHistoryRequest } from './journal-domain-history-page'
import type { JournalDomainImportState } from './journal-domain-import-rows'
export type JournalSqlReaderOperation =
| { kind: 'legacy-proof'; cursor: LegacyProofCursor }
| { kind: 'fork-prepare'; input: JournalForkPreparationInput }
| { kind: 'fork-next'; preparationId: number }
| { kind: 'fork-release'; preparationId: number }
@@ -27,6 +29,7 @@ export type JournalSqlReaderOperation =
export type JournalSqlReaderRequest = { id: number; operation: JournalSqlReaderOperation }
export type JournalSqlReaderReply =
| { type: 'legacy-proof'; id: number; page: LegacyProofPage }
| { type: 'fork-prepared'; id: number }
| { type: 'fork-next'; id: number; result: JournalForkPreparationResult }
| { type: 'ready' }
@@ -1,3 +1,4 @@
import { legacyDestinationMain, legacyDestinationIdentity } from './journal-legacy-retirement-files'
import { JournalForkStages } from './journal-fork-stages'
import { mkdirSync } from 'node:fs'
import { dirname } from 'node:path'
@@ -34,6 +35,7 @@ let commands: JournalSqlCommand | undefined
let forks: JournalForkStages | undefined
let checkpoint: JournalCheckpointOwner | undefined
let checkpointClosing: Promise<void> | undefined
let destinationMain: string | undefined
let readOnly = true
let fault: unknown
let closing = false
@@ -48,6 +50,11 @@ async function open(): Promise<void> {
}
const opened = openJournalDomainDatabase(workerData.filename)
db = opened.db
try {
destinationMain = legacyDestinationMain(workerData.filename)
} catch {
/* Cleanup eligibility never gates opening a chat. */
}
readOnly = opened.readOnly
forks = new JournalForkStages(db, stateDirectory)
commands = new JournalSqlCommand(db, stateDirectory, forks)
@@ -126,6 +133,16 @@ async function drain(): Promise<void> {
const checkpoint = forks!.checkpoint()
try {
if (command.type === 'maintenance') {
if (command.operation.kind === 'legacy-retirement-save') {
if (
destinationMain !== legacyDestinationMain(workerData.filename) ||
(command.operation.destination &&
JSON.stringify(command.operation.destination) !==
JSON.stringify(legacyDestinationIdentity(workerData.filename)))
) {
throw new Error('journal destination generation changed')
}
}
return {
type: 'maintenance',
id,
@@ -154,10 +171,17 @@ async function drain(): Promise<void> {
}
},
writes.some(
(request) => request.command.type === 'write' && request.command.command.forkInstall
(request) =>
request.command.type === 'maintenance' &&
request.command.operation.kind === 'legacy-retirement-save' &&
request.command.operation.durable
)
? 'full'
: 'normal'
? 'retirement'
: writes.some(
(request) => request.command.type === 'write' && request.command.command.forkInstall
)
? 'full'
: 'normal'
)
forks!.acknowledge()
// All acknowledged commits precede a fault notice; it never retracts a successful COMMIT.
@@ -1,4 +1,5 @@
// Holding the lock means re-proving it on every heartbeat, not only at acquisition.
import { dirname, relative, isAbsolute } from 'node:path'
import { statSync, unlinkSync, utimesSync } from 'node:fs'
import { startOrcadLockHeartbeat } from './orcad-lock-clone-machine'
import { readBoundedLockFile } from './orcad-instance-lock-reclaim'
@@ -18,6 +19,22 @@ export type OrcadInstanceLock = {
release(options?: { keepUnexpectedExit?: boolean }): void
}
const ownedProfileLocks = new Map<string, OrcadLockRecord>()
export function ownsOrcadProfileDirectory(directory: string): boolean {
for (const [path, record] of ownedProfileLocks) {
const child = relative(dirname(path), directory)
if (child.startsWith('..') || isAbsolute(child)) {
continue
}
const current = parseOrcadInstanceLockRecord(readBoundedLockFile(path) ?? '')
if (current?.nonce === record.nonce && current.pid === process.pid) {
return true
}
}
return false
}
type OrcadLockBeat = 'held' | 'lost' | 'unproven'
function lockIsAbsent(lockPath: string): boolean {
@@ -89,6 +106,7 @@ export function holdOrcadInstanceLock(
onExclusivityLost: () => void
}
): OrcadInstanceLock {
ownedProfileLocks.set(lockPath, record)
let released = false
const stopHeartbeat = startOrcadLockHeartbeat(() => {
if (beatOrcadInstanceLock(lockPath, record, publish) === 'lost') {
@@ -105,6 +123,9 @@ export function holdOrcadInstanceLock(
return
}
released = true
if (ownedProfileLocks.get(lockPath)?.nonce === record.nonce) {
ownedProfileLocks.delete(lockPath)
}
stopHeartbeat()
// Re-read ownership before unlinking so a successor’s live runtime cannot be unlocked.
const current = parseOrcadInstanceLockRecord(readBoundedLockFile(lockPath) ?? '')
+4 -1
View File
@@ -75,6 +75,7 @@ export const ORCAD_NODE_PTY_JS_ARTIFACTS = [
export function orcadNodePtySlotFiles(target: string): string[] {
if (isWindowsTarget(target)) {
return [
'journal-file-lock.exe',
'conpty.node',
'conpty_console_list.node',
'conpty/conpty.dll',
@@ -85,7 +86,9 @@ export function orcadNodePtySlotFiles(target: string): string[] {
'winpty-agent.exe'
]
}
return target.startsWith('darwin-') ? ['pty.node', 'spawn-helper'] : ['pty.node']
return target.startsWith('darwin-')
? ['journal-file-lock', 'pty.node', 'spawn-helper']
: ['journal-file-lock', 'pty.node']
}
export function orcadNodePtyNativeArtifacts(target: string): string[] {