mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
fix(native-chat): retire certified legacy journals after the grace period
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
name: Journal file lock
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'native/journal-file-lock/**'
|
||||
- 'config/scripts/native-build/build-journal-file-lock.mjs'
|
||||
- 'src/main/native-chat/agent-session-journal/journal-legacy*'
|
||||
- '.github/workflows/journal-file-lock.yml'
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
lock:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04, macos-15, windows-2025]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/install-node-dependencies
|
||||
with:
|
||||
native-runtime: node
|
||||
node-version: '24'
|
||||
- run: node config/scripts/native-build/build-journal-file-lock.mjs
|
||||
- run: node node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/native-chat/agent-session-journal/journal-legacy-file-lock.test.ts
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
@@ -166,6 +166,11 @@ const emojiShortcodeDatasetResource = {
|
||||
to: 'node_modules/emojibase-data/en/shortcodes/emojibase.json'
|
||||
}
|
||||
const commonExtraResources = [
|
||||
{
|
||||
from: 'node_modules/node-pty/build/Release',
|
||||
to: 'bin',
|
||||
filter: ['journal-file-lock', 'journal-file-lock.exe']
|
||||
},
|
||||
relayExtraResource,
|
||||
orcadTemplateExtraResource,
|
||||
orcadTemplateNodeModulesExtraResource,
|
||||
@@ -408,6 +413,16 @@ module.exports = {
|
||||
if (context.electronPlatformName === 'linux') {
|
||||
ensureWaylandClipboard(context.arch)
|
||||
}
|
||||
return import('./scripts/native-build/build-journal-file-lock.mjs').then(
|
||||
({ buildJournalFileLock }) => {
|
||||
const lockName =
|
||||
context.electronPlatformName === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
|
||||
buildJournalFileLock(
|
||||
join(__dirname, '..', 'node_modules', 'node-pty', 'build', 'Release', lockName),
|
||||
context.arch === 3 ? 'arm64' : 'x64'
|
||||
)
|
||||
}
|
||||
)
|
||||
},
|
||||
afterPack: async (context) => {
|
||||
const resourcesDir =
|
||||
@@ -537,6 +552,11 @@ module.exports = {
|
||||
})
|
||||
chmodUnixCliLaunchers(resourcesDir, context.electronPlatformName)
|
||||
if (context.electronPlatformName === 'darwin') {
|
||||
await signMacStandaloneHelper(
|
||||
join(resourcesDir, 'bin', 'journal-file-lock'),
|
||||
'journal file lock',
|
||||
context.packager
|
||||
)
|
||||
await signMacComputerUseHelper(join(resourcesDir, 'Orca Computer Use.app'), context.packager)
|
||||
await signMacStandaloneHelper(
|
||||
join(resourcesDir, '..', 'MacOS', 'orca-notification-status'),
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import {
|
||||
initializeWindowsNativeToolchain,
|
||||
mergeWindowsCompilerEnvironment
|
||||
} from './windows/initialize-windows-native-toolchain.mjs'
|
||||
import { runProcessSync } from '@orca/process-host'
|
||||
|
||||
export function buildJournalFileLock(output, arch = process.arch) {
|
||||
const root = resolve(import.meta.dirname, '../../..')
|
||||
mkdirSync(dirname(output), { recursive: true })
|
||||
const source = resolve(root, 'native/journal-file-lock/main.c')
|
||||
const windows = process.platform === 'win32'
|
||||
const args = windows
|
||||
? [
|
||||
'/nologo',
|
||||
'/O2',
|
||||
'/W4',
|
||||
'/WX',
|
||||
'/DUNICODE',
|
||||
'/D_UNICODE',
|
||||
source,
|
||||
`/Fe:${output}`,
|
||||
`/Fo:${output}.obj`
|
||||
]
|
||||
: [
|
||||
'-O2',
|
||||
'-Wall',
|
||||
'-Wextra',
|
||||
'-Werror',
|
||||
...(process.platform === 'darwin' ? ['-arch', arch === 'x64' ? 'x86_64' : arch] : []),
|
||||
source,
|
||||
'-o',
|
||||
output
|
||||
]
|
||||
if (process.platform === 'linux' && arch !== process.arch) {
|
||||
throw new Error('native lock needs a target-native Linux builder')
|
||||
}
|
||||
if (windows && arch !== process.arch) {
|
||||
throw new Error('native lock needs a target-native Windows builder')
|
||||
}
|
||||
const compilerEnvironment = windows ? initializeWindowsNativeToolchain() : undefined
|
||||
const result = runProcessSync({
|
||||
program: windows ? 'cl.exe' : 'cc',
|
||||
args,
|
||||
cwd: root,
|
||||
env: compilerEnvironment
|
||||
? mergeWindowsCompilerEnvironment(process.env, compilerEnvironment)
|
||||
: process.env,
|
||||
timeoutMs: 120000
|
||||
})
|
||||
if (result.code !== 0) {
|
||||
throw new Error(`journal lock build failed: ${result.stderr}${result.stdout}`)
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
buildJournalFileLock(
|
||||
resolve(
|
||||
'native/journal-file-lock/.build',
|
||||
process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
|
||||
)
|
||||
)
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env node
|
||||
import { buildJournalFileLock } from '../../native-build/build-journal-file-lock.mjs'
|
||||
// Build patched node-pty for current platform/arch/libc so deployment needs no compiler; compat slots also build COMPAT_SLOT_ADDONS.
|
||||
// Require repo patch: GLIBC symbol pins plus no-as-needed retain libutil/libpthread and prevent #9902.
|
||||
// Use pinned, hash-verified Node headers at N-API 8; glibc slots target 2.28, optional compat 2.17, musl stays separate.
|
||||
@@ -254,6 +255,9 @@ async function build() {
|
||||
? await compileCompatAddons({ slot, workDir: join(WORK_DIR, slot), nodeDir })
|
||||
: []
|
||||
|
||||
buildJournalFileLock(
|
||||
join(buildDir, process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock')
|
||||
)
|
||||
rmSync(slotDir, { recursive: true, force: true })
|
||||
const files = {}
|
||||
for (const [relative, source] of [
|
||||
|
||||
@@ -95,6 +95,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) {
|
||||
const conptyRuntime = windowsConptyRuntimeDir(nodePtyDir, arch)
|
||||
const upstream = join(nodePtyDir, 'prebuilds', `win32-${arch}`)
|
||||
return [
|
||||
['journal-file-lock.exe', join(buildDir, 'journal-file-lock.exe')],
|
||||
['conpty.node', join(buildDir, 'conpty.node')],
|
||||
['conpty_console_list.node', join(upstream, 'conpty_console_list.node')],
|
||||
['conpty/conpty.dll', join(conptyRuntime, 'conpty.dll')],
|
||||
@@ -104,6 +105,7 @@ export function slotSourceFiles({ platform, arch, buildDir, nodePtyDir }) {
|
||||
}
|
||||
// macOS posix_spawns spawn-helper and builds it only there; Linux forks directly.
|
||||
return [
|
||||
['journal-file-lock', join(buildDir, 'journal-file-lock')],
|
||||
['pty.node', join(buildDir, 'pty.node')],
|
||||
...(platform === 'darwin' ? [['spawn-helper', join(buildDir, 'spawn-helper')]] : [])
|
||||
]
|
||||
|
||||
@@ -23,6 +23,8 @@ export const WINDOWS_PREBUILD_LOCKED_PACKAGES = ['node-pty', 'node-gyp']
|
||||
const WINDOWS_PREBUILD_PACKAGE_SCRIPTS = ['build:orcad-prebuilds', 'build:packages']
|
||||
export const WINDOWS_PREBUILD_CACHE_INPUTS = [
|
||||
'.github/actions/prepare-orcad-prebuilds/action.yml',
|
||||
'native/journal-file-lock/main.c',
|
||||
'config/scripts/native-build/build-journal-file-lock.mjs',
|
||||
'config/patches/node-pty@1.1.0.patch',
|
||||
'src/shared/node-runtime-pin.ts',
|
||||
'config/scripts/orcad-distribution/prebuilds/build-orcad-prebuilds.mjs',
|
||||
|
||||
@@ -38,6 +38,7 @@ function fixture(arch = 'x64') {
|
||||
mkdirSync(join(sourceDir, 'third_party/conpty/1', `win10-${arch}`), { recursive: true })
|
||||
const files = {}
|
||||
for (const [file, bytes] of [
|
||||
['journal-file-lock.exe', peImage({ arch })],
|
||||
[
|
||||
'conpty.node',
|
||||
Buffer.concat([peImage({ arch }), CYGWIN_BREAKAWAY_MARKER, NATIVE_HOST_JOB_MARKER])
|
||||
@@ -51,7 +52,7 @@ function fixture(arch = 'x64') {
|
||||
]) {
|
||||
write(join(slotDir, file), bytes)
|
||||
files[file] = sha256Of(join(slotDir, file))
|
||||
if (file !== 'conpty.node' && !file.startsWith('conpty/')) {
|
||||
if (file !== 'journal-file-lock.exe' && file !== 'conpty.node' && !file.startsWith('conpty/')) {
|
||||
write(join(sourceDir, 'prebuilds', slot, file), bytes)
|
||||
}
|
||||
if (file.startsWith('conpty/')) {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
.build/
|
||||
@@ -0,0 +1,73 @@
|
||||
/* Holds the standard SQLite main-file lock without opening or modifying SQLite. */
|
||||
#ifdef _WIN32
|
||||
#define _WIN32_WINNT 0x0602
|
||||
#include <windows.h>
|
||||
#include <stdio.h>
|
||||
#include <wchar.h>
|
||||
int wmain(int argc, wchar_t **argv) {
|
||||
if (argc != 2) return 2;
|
||||
HANDLE file = CreateFileW(argv[1], GENERIC_READ | DELETE, FILE_SHARE_DELETE,
|
||||
NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL);
|
||||
if (file == INVALID_HANDLE_VALUE) return 3;
|
||||
BY_HANDLE_FILE_INFORMATION info;
|
||||
wchar_t fs[32], volume[MAX_PATH], final[32768];
|
||||
DWORD final_length = GetFinalPathNameByHandleW(file, final, 32768, FILE_NAME_NORMALIZED);
|
||||
if (!GetFileInformationByHandle(file, &info) || info.nNumberOfLinks != 1 ||
|
||||
(info.dwFileAttributes & (FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT)) ||
|
||||
final_length == 0 || final_length >= 32768 ||
|
||||
!GetVolumePathNameW(final, volume, MAX_PATH) || GetDriveTypeW(volume) != DRIVE_FIXED ||
|
||||
!GetVolumeInformationByHandleW(file, NULL, 0, NULL, NULL, NULL, fs, 32) ||
|
||||
wcscmp(fs, L"NTFS") != 0) {
|
||||
CloseHandle(file); return 4;
|
||||
}
|
||||
HANDLE named = CreateFileW(argv[1], 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
|
||||
NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL);
|
||||
BY_HANDLE_FILE_INFORMATION current;
|
||||
int same = named != INVALID_HANDLE_VALUE && GetFileInformationByHandle(named, ¤t) &&
|
||||
current.dwVolumeSerialNumber == info.dwVolumeSerialNumber &&
|
||||
current.nFileIndexHigh == info.nFileIndexHigh && current.nFileIndexLow == info.nFileIndexLow;
|
||||
if (named != INVALID_HANDLE_VALUE && !CloseHandle(named)) same = 0;
|
||||
if (!same) { CloseHandle(file); return 4; }
|
||||
puts("locked"); fflush(stdout);
|
||||
while (getchar() != EOF) {}
|
||||
return CloseHandle(file) ? 0 : 5;
|
||||
}
|
||||
#else
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#ifdef __APPLE__
|
||||
#include <sys/mount.h>
|
||||
#else
|
||||
#include <sys/vfs.h>
|
||||
#endif
|
||||
int main(int argc, char **argv) {
|
||||
if (argc != 2) return 2;
|
||||
int fd = open(argv[1], O_RDWR | O_CLOEXEC | O_NOFOLLOW);
|
||||
if (fd < 0) return 3;
|
||||
struct stat info, named;
|
||||
struct statfs volume;
|
||||
int local = 0;
|
||||
if (fstatfs(fd, &volume) == 0) {
|
||||
#ifdef __APPLE__
|
||||
local = strcmp(volume.f_fstypename, "apfs") == 0 || strcmp(volume.f_fstypename, "hfs") == 0;
|
||||
#else
|
||||
local = volume.f_type == 0xef53 || volume.f_type == 0x58465342 ||
|
||||
volume.f_type == 0x9123683e || volume.f_type == 0x2fc12fc1;
|
||||
#endif
|
||||
}
|
||||
struct flock lock = {0};
|
||||
lock.l_type = F_WRLCK; lock.l_whence = SEEK_SET;
|
||||
lock.l_start = 0x40000000; lock.l_len = 512;
|
||||
if (!local || fstat(fd, &info) || !S_ISREG(info.st_mode) || info.st_nlink != 1 ||
|
||||
fcntl(fd, F_SETLK, &lock) || lstat(argv[1], &named) ||
|
||||
info.st_dev != named.st_dev || info.st_ino != named.st_ino) {
|
||||
close(fd); return 4;
|
||||
}
|
||||
puts("locked"); fflush(stdout);
|
||||
while (getchar() != EOF) {}
|
||||
return close(fd) == 0 ? 0 : 5;
|
||||
}
|
||||
#endif
|
||||
@@ -61,6 +61,7 @@ export class JournalSqlDatabase {
|
||||
this.activity,
|
||||
options.onMaintenanceError
|
||||
)
|
||||
this.maintenance.ownLegacySource(stateDirectory, this.readers)
|
||||
}
|
||||
|
||||
static async open(
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { captureJournalLegacySourceReceipt } from './database/journal-legacy-source-receipt'
|
||||
import { existsSync, mkdtempSync, rmSync, writeFileSync, readFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { spawnProcess } from '@orca/process-host'
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { acquireJournalLegacyFileLock, journalLegacyFileLockPath } from './journal-legacy-file-lock'
|
||||
import { openJournalDatabase } from './database/journal-database'
|
||||
|
||||
const cleanups: (() => void | Promise<void>)[] = []
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0).toReversed()) {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
|
||||
it.skipIf(!process.env.CI)(
|
||||
'excludes a real SQLite process without altering source bytes, then releases on EOF',
|
||||
async () => {
|
||||
expect(existsSync(journalLegacyFileLockPath())).toBe(true)
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-journal-file-lock-'))
|
||||
cleanups.push(() => rmSync(root, { recursive: true, force: true }))
|
||||
const filename = join(root, 'source.db')
|
||||
const { db } = openJournalDatabase(filename)
|
||||
db.exec(
|
||||
"CREATE TABLE fixture_history(value TEXT); INSERT INTO fixture_history VALUES('history never opened')"
|
||||
)
|
||||
db.close()
|
||||
const script = join(root, 'reader.cjs')
|
||||
writeFileSync(
|
||||
script,
|
||||
"const{DatabaseSync}=require('node:sqlite');const db=new DatabaseSync(process.argv[2], {readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('ready');process.stdin.resume();process.stdin.on('end',()=>{db.close()});"
|
||||
)
|
||||
const reader = spawnProcess({
|
||||
program: process.execPath,
|
||||
args: [script, filename],
|
||||
stdio: 'pipe'
|
||||
})
|
||||
const exited = new Promise<void>((resolve, reject) => {
|
||||
reader.once('error', reject)
|
||||
reader.once('exit', () => resolve())
|
||||
})
|
||||
cleanups.push(async () => {
|
||||
reader.stdin.end()
|
||||
await exited
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
reader.stdout.once('data', () => resolve())
|
||||
reader.once('error', reject)
|
||||
})
|
||||
const signal = new AbortController()
|
||||
await expect(acquireJournalLegacyFileLock(filename, signal.signal)).rejects.toThrow()
|
||||
reader.stdin.end()
|
||||
await exited
|
||||
const before = readFileSync(filename)
|
||||
const identity = captureJournalLegacySourceReceipt(filename)
|
||||
const lock = await acquireJournalLegacyFileLock(filename, signal.signal)
|
||||
try {
|
||||
expect(lock.held()).toBe(true)
|
||||
expect(captureJournalLegacySourceReceipt(filename)).toEqual(identity)
|
||||
const contender = spawnProcess({
|
||||
program: process.execPath,
|
||||
args: [
|
||||
'-e',
|
||||
"const{DatabaseSync}=require('node:sqlite');let db;try{db=new DatabaseSync(process.argv[1],{readOnly:true});db.prepare('SELECT * FROM fixture_history').get();process.stdout.write('opened')}catch{process.stdout.write('blocked')}finally{db?.close()}",
|
||||
filename
|
||||
],
|
||||
stdio: 'pipe'
|
||||
})
|
||||
contender.stdin.end()
|
||||
const output = await new Promise<string>((resolve, reject) => {
|
||||
let result = ''
|
||||
contender.stdout.on('data', (chunk) => {
|
||||
result += chunk.toString()
|
||||
})
|
||||
contender.stderr.resume()
|
||||
contender.once('error', reject)
|
||||
contender.once('exit', () => resolve(result))
|
||||
})
|
||||
expect(output).toBe('blocked')
|
||||
} finally {
|
||||
await lock.close()
|
||||
}
|
||||
expect(readFileSync(filename)).toEqual(before)
|
||||
const second = await acquireJournalLegacyFileLock(filename, signal.signal)
|
||||
await second.close()
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
import { spawnProcess } from '@orca/process-host'
|
||||
import { createRequire } from 'node:module'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { currentWorkerEntryLayout } from '../../worker-threads/worker-thread-entry-path'
|
||||
|
||||
export function journalLegacyFileLockPath(): string {
|
||||
const name = process.platform === 'win32' ? 'journal-file-lock.exe' : 'journal-file-lock'
|
||||
const layout = currentWorkerEntryLayout(__dirname)
|
||||
if (layout.isPackaged && layout.resourcesPath) {
|
||||
return join(layout.resourcesPath, 'bin', name)
|
||||
}
|
||||
const require = createRequire(__filename)
|
||||
const staged = join(dirname(require.resolve('node-pty/package.json')), 'build', 'Release', name)
|
||||
return existsSync(staged) ? staged : resolve('native/journal-file-lock/.build', name)
|
||||
}
|
||||
|
||||
/** The helper never opens SQLite; closing stdin releases its single non-writing source handle. */
|
||||
export async function acquireJournalLegacyFileLock(filename: string, signal: AbortSignal) {
|
||||
signal.throwIfAborted()
|
||||
const child = spawnProcess({
|
||||
program: journalLegacyFileLockPath(),
|
||||
args: [filename],
|
||||
stdio: 'pipe'
|
||||
})
|
||||
child.stdin.on('error', () => {})
|
||||
child.stderr.resume()
|
||||
let exited = false
|
||||
const done = new Promise<void>((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code) => {
|
||||
exited = true
|
||||
if (code === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new Error('legacy journal lock is unavailable'))
|
||||
}
|
||||
})
|
||||
})
|
||||
void done.catch(() => {})
|
||||
const release = () => child.stdin.end()
|
||||
signal.addEventListener('abort', release, { once: true })
|
||||
if (signal.aborted) {
|
||||
release()
|
||||
}
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let output = ''
|
||||
child.stdout.on('data', (chunk: Buffer) => {
|
||||
output += chunk.toString()
|
||||
if (output === 'locked\n' || output === 'locked\r\n') {
|
||||
resolve()
|
||||
} else if (output.length > 64) {
|
||||
reject(new Error('invalid legacy lock acknowledgement'))
|
||||
}
|
||||
})
|
||||
child.once('error', reject)
|
||||
child.once('exit', () => reject(new Error('legacy journal lock was refused')))
|
||||
})
|
||||
} catch (error) {
|
||||
release()
|
||||
await done.catch(() => {})
|
||||
signal.removeEventListener('abort', release)
|
||||
throw error
|
||||
}
|
||||
return {
|
||||
held: () => !exited && !signal.aborted,
|
||||
close: async () => {
|
||||
release()
|
||||
try {
|
||||
await done
|
||||
} finally {
|
||||
signal.removeEventListener('abort', release)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
import { lstat, open, stat, unlink } from 'node:fs/promises'
|
||||
import { lstatSync } from 'node:fs'
|
||||
import { dirname } from 'node:path'
|
||||
import { journalDomainDatabasePath } from './database/journal-domain-database'
|
||||
import {
|
||||
journalLegacyFileIdentity,
|
||||
type JournalLegacyFileIdentity
|
||||
} from './database/journal-legacy-source-receipt'
|
||||
|
||||
export const LEGACY_COMPANION_SUFFIXES = ['-wal', '-shm'] as const
|
||||
export type LegacyCompanionManifest = Partial<
|
||||
Record<(typeof LEGACY_COMPANION_SUFFIXES)[number], JournalLegacyFileIdentity>
|
||||
>
|
||||
export type LegacyDestinationIdentity = { main: string; wal: string }
|
||||
export function legacyDestinationMain(filename: string): string {
|
||||
const file = lstatSync(filename, { bigint: true })
|
||||
if (!file.isFile() || file.nlink !== 1n || file.ino === 0n) {
|
||||
throw new Error('invalid journal destination identity')
|
||||
}
|
||||
return `${file.dev}:${file.ino}`
|
||||
}
|
||||
export function legacyDestinationIdentity(filename: string): LegacyDestinationIdentity {
|
||||
return { main: legacyDestinationMain(filename), wal: legacyDestinationMain(`${filename}-wal`) }
|
||||
}
|
||||
|
||||
export async function legacyFilePresent(filename: string): Promise<boolean> {
|
||||
try {
|
||||
await lstat(filename)
|
||||
return true
|
||||
} catch (error) {
|
||||
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
|
||||
return false
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncLegacyDirectory(directory: string): Promise<void> {
|
||||
if (process.platform === 'win32') {
|
||||
return
|
||||
}
|
||||
// Ancestors are synced bottom-up so a newly created profile directory is covered too.
|
||||
for (;;) {
|
||||
const handle = await open(directory, 'r')
|
||||
try {
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
const parent = dirname(directory)
|
||||
if (parent === directory) {
|
||||
return
|
||||
}
|
||||
directory = parent
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncLegacyDestination(directory: string, source: string): Promise<void> {
|
||||
const domain = journalDomainDatabasePath(directory)
|
||||
const sourceFile = await stat(source, { bigint: true })
|
||||
for (const name of [domain, `${domain}-wal`]) {
|
||||
const file = await lstat(name, { bigint: true })
|
||||
if (
|
||||
!file.isFile() ||
|
||||
file.dev !== sourceFile.dev ||
|
||||
file.ino === sourceFile.ino ||
|
||||
file.nlink !== 1n
|
||||
) {
|
||||
throw new Error('journal destination volume or identity changed')
|
||||
}
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
const sourceDevice = sourceFile.dev
|
||||
for (const name of [domain, `${domain}-wal`]) {
|
||||
const handle = await open(name, 'r+')
|
||||
try {
|
||||
if ((await handle.stat({ bigint: true })).dev !== sourceDevice) {
|
||||
throw new Error('journal destination volume changed')
|
||||
}
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
await syncLegacyDirectory(directory)
|
||||
}
|
||||
|
||||
export async function removeLegacyCompanions(
|
||||
directory: string,
|
||||
filename: string,
|
||||
manifest: LegacyCompanionManifest | undefined,
|
||||
boundary: () => Promise<boolean>,
|
||||
ownsProfile: () => boolean,
|
||||
signal: AbortSignal
|
||||
): Promise<boolean> {
|
||||
if (!manifest || !ownsProfile() || signal.aborted) {
|
||||
return false
|
||||
}
|
||||
for (const suffix of LEGACY_COMPANION_SUFFIXES) {
|
||||
if (
|
||||
!(await boundary()) ||
|
||||
!ownsProfile() ||
|
||||
(await legacyFilePresent(filename)) ||
|
||||
(await legacyFilePresent(`${filename}-journal`))
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const name = filename + suffix
|
||||
if (!(await legacyFilePresent(name))) {
|
||||
continue
|
||||
}
|
||||
const expected = manifest[suffix]
|
||||
const current = await lstat(name)
|
||||
if (
|
||||
!current.isFile() ||
|
||||
!expected ||
|
||||
JSON.stringify(journalLegacyFileIdentity(name)) !== JSON.stringify(expected) ||
|
||||
(suffix === '-wal' && expected.size !== '0')
|
||||
) {
|
||||
throw new Error('legacy companion changed; retaining it')
|
||||
}
|
||||
await unlink(name)
|
||||
}
|
||||
if (
|
||||
(await legacyFilePresent(filename)) ||
|
||||
(
|
||||
await Promise.all(
|
||||
LEGACY_COMPANION_SUFFIXES.map((suffix) => legacyFilePresent(filename + suffix))
|
||||
)
|
||||
).some(Boolean)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
await syncLegacyDirectory(directory)
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import type { LegacyCompanionManifest } from './journal-legacy-retirement-files'
|
||||
|
||||
export const LEGACY_RETIREMENT_WEEK = 7 * 24 * 60 * 60 * 1000
|
||||
export type Retirement = {
|
||||
version: 1
|
||||
proof: string
|
||||
proofId: string
|
||||
completedAt: number
|
||||
process: string
|
||||
credit: number
|
||||
creditRevision: number
|
||||
lastWall: number
|
||||
phase: 'grace' | 'prepared' | 'complete'
|
||||
manifest?: LegacyCompanionManifest
|
||||
}
|
||||
|
||||
export function decodeRetirement(value: string | null): Retirement | null {
|
||||
if (!value) {
|
||||
return null
|
||||
}
|
||||
const data: Retirement = JSON.parse(value)
|
||||
if (
|
||||
data.version !== 1 ||
|
||||
typeof data.proof !== 'string' ||
|
||||
typeof data.proofId !== 'string' ||
|
||||
typeof data.process !== 'string' ||
|
||||
!['grace', 'prepared', 'complete'].includes(data.phase) ||
|
||||
!Number.isFinite(data.credit) ||
|
||||
data.credit < 0 ||
|
||||
data.credit > LEGACY_RETIREMENT_WEEK ||
|
||||
!Number.isSafeInteger(data.creditRevision) ||
|
||||
data.creditRevision < 0
|
||||
) {
|
||||
throw new Error('invalid legacy retirement receipt')
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
export function advanceLegacyRetirementClock(
|
||||
state: Retirement,
|
||||
now: number,
|
||||
elapsed: number,
|
||||
processToken: string
|
||||
): Retirement {
|
||||
if (
|
||||
!Number.isFinite(state.completedAt) ||
|
||||
!Number.isFinite(state.lastWall) ||
|
||||
state.completedAt < 0 ||
|
||||
state.lastWall < 0 ||
|
||||
state.completedAt > now ||
|
||||
state.lastWall > now
|
||||
) {
|
||||
return {
|
||||
...state,
|
||||
completedAt: now,
|
||||
lastWall: now,
|
||||
credit: 0,
|
||||
process: processToken,
|
||||
phase: 'grace'
|
||||
}
|
||||
}
|
||||
return {
|
||||
...state,
|
||||
credit: Math.min(
|
||||
LEGACY_RETIREMENT_WEEK,
|
||||
state.credit + Math.max(0, Number.isFinite(elapsed) ? elapsed : 0)
|
||||
),
|
||||
lastWall: now
|
||||
}
|
||||
}
|
||||
|
||||
/** Absolute revisions discard ambiguous saved credit instead of counting one interval twice. */
|
||||
export class JournalLegacyGraceClock {
|
||||
private anchor?: bigint
|
||||
private revision?: string
|
||||
private pending = 0
|
||||
|
||||
observe(state: Retirement, now: number, processToken: string): Retirement {
|
||||
const revision = `${state.proofId}:${state.creditRevision}`
|
||||
if (this.revision !== revision) {
|
||||
this.pending = 0
|
||||
this.anchor = undefined
|
||||
this.revision = revision
|
||||
}
|
||||
const mono = process.hrtime.bigint()
|
||||
if (this.anchor !== undefined && mono >= this.anchor) {
|
||||
this.pending += Number(mono - this.anchor) / 1e6
|
||||
}
|
||||
this.anchor = mono
|
||||
return advanceLegacyRetirementClock(state, now, this.pending, processToken)
|
||||
}
|
||||
|
||||
persisted(state: Retirement): void {
|
||||
this.revision = `${state.proofId}:${state.creditRevision}`
|
||||
this.pending = 0
|
||||
this.anchor = process.hrtime.bigint()
|
||||
}
|
||||
|
||||
pause(): void {
|
||||
const mono = process.hrtime.bigint()
|
||||
if (this.anchor !== undefined && mono >= this.anchor) {
|
||||
this.pending += Number(mono - this.anchor) / 1e6
|
||||
}
|
||||
this.anchor = undefined
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
import {
|
||||
mkdtempSync,
|
||||
rmSync,
|
||||
existsSync,
|
||||
writeFileSync,
|
||||
readFileSync,
|
||||
statSync,
|
||||
readdirSync,
|
||||
unlinkSync,
|
||||
statfsSync
|
||||
} from 'node:fs'
|
||||
import { join, basename } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { openJournalDatabase } from './database/journal-database'
|
||||
import {
|
||||
initializeJournalDomainDatabase,
|
||||
openJournalDomainDatabase
|
||||
} from './database/journal-domain-database'
|
||||
import { journalLegacyFileIdentity } from './database/journal-legacy-source-receipt'
|
||||
import { JournalSqlDatabase } from './database/journal-sql-database'
|
||||
import {
|
||||
journalSqlWorkerPath,
|
||||
journalSqlReaderPath
|
||||
} from './journal-sql-worker-bundle.test-fixture'
|
||||
import { message } from './journal-sql-message.test-fixture'
|
||||
import { journalDatabasePath } from './database/journal-host-database'
|
||||
import { JournalSqlBatch } from './journal-sql-batch'
|
||||
import { JournalLegacyRetirement } from './journal-legacy-retirement'
|
||||
import {
|
||||
advanceLegacyRetirementClock,
|
||||
JournalLegacyGraceClock
|
||||
} from './journal-legacy-retirement-state'
|
||||
import { readJournalLegacyProofPage } from './journal-legacy-source-proof'
|
||||
import { applyJournalSqlMaintenance, type JournalSqlMaintenance } from './journal-sql-maintenance'
|
||||
import { runJournalTransaction } from './database/journal-database'
|
||||
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
|
||||
import { journalDomainImportState } from './journal-domain-import-rows'
|
||||
import { prepareJournalDomainImport } from './journal-domain-import'
|
||||
import { journalSqlRetryRows } from './journal-sql-retry-rows.test-fixture'
|
||||
import { insertJournalRow, publishJournalSessionEpoch } from './durable-rows/journal-row-table'
|
||||
|
||||
const cleanup: (() => void)[] = []
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
for (const close of cleanup.splice(0).toReversed()) {
|
||||
close()
|
||||
}
|
||||
})
|
||||
|
||||
async function profile(seed?: (db: Database.Database) => void) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-legacy-retirement-'))
|
||||
cleanup.push(() => rmSync(root, { recursive: true, force: true }))
|
||||
const filename = journalDatabasePath(root)
|
||||
const { db: legacy } = openJournalDatabase(filename)
|
||||
runJournalTransaction(legacy, () => {
|
||||
publishJournalSessionEpoch(
|
||||
legacy,
|
||||
{ sessionId: 'unopened', workspaceId: 'folder-without-git' },
|
||||
'epoch'
|
||||
)
|
||||
for (const row of journalSqlRetryRows(true)) {
|
||||
insertJournalRow(legacy, 'unopened', row)
|
||||
}
|
||||
})
|
||||
seed?.(legacy)
|
||||
legacy.close()
|
||||
const { db } = openJournalDomainDatabase(await initializeJournalDomainDatabase(root))
|
||||
let closed = false
|
||||
const closeDomain = () => {
|
||||
if (!closed) {
|
||||
db.close()
|
||||
closed = true
|
||||
}
|
||||
}
|
||||
cleanup.push(closeDomain)
|
||||
let original: Database.Database | undefined
|
||||
const source = () =>
|
||||
(original ??= new Database(filename, { readonly: true, fileMustExist: true }))
|
||||
cleanup.push(() => original?.close())
|
||||
const batch = new JournalSqlBatch(db)
|
||||
const writer = {
|
||||
maintain: async (operation: JournalSqlMaintenance) => {
|
||||
const result = await batch.run(
|
||||
[operation],
|
||||
(unit) => applyJournalSqlMaintenance(db, unit),
|
||||
operation.kind === 'legacy-retirement-save' && operation.durable ? 'retirement' : 'normal'
|
||||
)
|
||||
const outcome = result.outcomes[0]!
|
||||
if (!outcome.committed) {
|
||||
throw outcome.error
|
||||
}
|
||||
return outcome.value
|
||||
}
|
||||
}
|
||||
const readers = {
|
||||
legacyProof: async (cursor) => readJournalLegacyProofPage(source(), cursor, db),
|
||||
importState: async (id) => journalDomainImportState(db, id),
|
||||
recycleSourceReaders: async () => {
|
||||
original?.close()
|
||||
original = undefined
|
||||
return new AbortController()
|
||||
},
|
||||
releaseSourceReaders: () => {}
|
||||
} satisfies Pick<
|
||||
JournalSqlReaderPool,
|
||||
'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders'
|
||||
>
|
||||
const conversions = {
|
||||
ensure: async (id: string) => {
|
||||
if (journalDomainImportState(db, id).state === 'ready') {
|
||||
return 'epoch'
|
||||
}
|
||||
const imported = prepareJournalDomainImport(source(), id)
|
||||
if (!imported) {
|
||||
return null
|
||||
}
|
||||
for (const page of imported.pages) {
|
||||
await writer.maintain({ kind: 'import-page', target: imported, page })
|
||||
}
|
||||
return String(await writer.maintain({ kind: 'import-finish', header: imported }))
|
||||
}
|
||||
}
|
||||
const errors: unknown[] = []
|
||||
const lock = async () => ({ held: () => true, close: async () => {} })
|
||||
const retirement = new JournalLegacyRetirement(
|
||||
root,
|
||||
writer,
|
||||
readers,
|
||||
conversions,
|
||||
async () => true,
|
||||
(error) => errors.push(error),
|
||||
() => true,
|
||||
lock
|
||||
)
|
||||
const receiptState = () =>
|
||||
JSON.parse(
|
||||
String(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'"
|
||||
)
|
||||
.get()!.value
|
||||
)
|
||||
)
|
||||
const saveState = (state: unknown) =>
|
||||
db
|
||||
.prepare(
|
||||
"UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'"
|
||||
)
|
||||
.run(JSON.stringify(state))
|
||||
const arm = () => {
|
||||
const row = db
|
||||
.prepare(
|
||||
"SELECT value FROM agent_session_store_meta WHERE key='journal.legacy-retirement.v1'"
|
||||
)
|
||||
.get()
|
||||
const state = JSON.parse(String(row!.value))
|
||||
state.credit = 604800000
|
||||
state.process = 'prior-process'
|
||||
db.prepare(
|
||||
"UPDATE agent_session_store_meta SET value=? WHERE key='journal.legacy-retirement.v1'"
|
||||
).run(JSON.stringify(state))
|
||||
}
|
||||
return {
|
||||
root,
|
||||
filename,
|
||||
db,
|
||||
retirement,
|
||||
errors,
|
||||
arm,
|
||||
source,
|
||||
receiptState,
|
||||
saveState,
|
||||
writer,
|
||||
closeDomain,
|
||||
recycle: readers.recycleSourceReaders
|
||||
}
|
||||
}
|
||||
|
||||
it('accounts for an unopened folder chat, removes only the certified file and keeps current history', async () => {
|
||||
const test = await profile()
|
||||
const allocated = () => ({
|
||||
files: Object.fromEntries(
|
||||
readdirSync(test.root).map((name) => {
|
||||
const file = statSync(join(test.root, name))
|
||||
return [name, { logical: Number(file.size), allocated: Number(file.blocks) * 512 }]
|
||||
})
|
||||
),
|
||||
freeBytes: Number(statfsSync(test.root).bfree) * Number(statfsSync(test.root).bsize)
|
||||
})
|
||||
const before = allocated()
|
||||
const proofStart = performance.now()
|
||||
await test.retirement.run()
|
||||
expect(test.errors).toEqual([])
|
||||
expect(existsSync(test.filename)).toBe(true)
|
||||
const proofMs = performance.now() - proofStart
|
||||
const proof = test.receiptState()
|
||||
test.saveState({ ...proof, credit: 604800000 })
|
||||
await test.retirement.run()
|
||||
expect(existsSync(test.filename)).toBe(true)
|
||||
test.arm()
|
||||
const deletionStart = performance.now()
|
||||
await test.retirement.run()
|
||||
expect(test.errors).toEqual([])
|
||||
expect(existsSync(test.filename)).toBe(false)
|
||||
expect(
|
||||
test.db
|
||||
.prepare(
|
||||
"SELECT payload_json FROM journal_items WHERE session_id='unopened' AND payload_json LIKE '%compact%' LIMIT 1"
|
||||
)
|
||||
.get()
|
||||
).toBeDefined()
|
||||
await test.retirement.run()
|
||||
expect(existsSync(test.filename)).toBe(false)
|
||||
test.db.pragma('wal_checkpoint(TRUNCATE)')
|
||||
test.closeDomain()
|
||||
const after = allocated()
|
||||
const total = (files: typeof before.files) =>
|
||||
Object.values(files).reduce((sum, value) => sum + value.allocated, 0)
|
||||
console.info('synthetic allocated bytes after checkpoint and handle closure', {
|
||||
before,
|
||||
after,
|
||||
grossLegacyRemoved: before.files[basename(test.filename)]!.allocated,
|
||||
netRemoved: total(before.files) - total(after.files),
|
||||
proofMs,
|
||||
retireAndCheckpointMs: performance.now() - deletionStart
|
||||
})
|
||||
const options = {
|
||||
writerPath: journalSqlWorkerPath(),
|
||||
readerPath: journalSqlReaderPath(),
|
||||
onMaintenanceError: () => {}
|
||||
}
|
||||
const current = await JournalSqlDatabase.open(test.root, options)
|
||||
try {
|
||||
const { target } = await current.openSession({
|
||||
sessionId: 'new-current-chat',
|
||||
workspaceId: 'folder-without-git',
|
||||
hostId: 'synthetic-host',
|
||||
agent: 'codex',
|
||||
providerHandle: null
|
||||
})
|
||||
const step = message('saved after legacy retirement')
|
||||
await current.write({
|
||||
target,
|
||||
savedAt: 100,
|
||||
operation: {
|
||||
kind: 'steps',
|
||||
steps: [
|
||||
{
|
||||
kind: 'resolved-item',
|
||||
reservedBytes: 1024,
|
||||
resolution: { kind: 'text-snapshot', identity: step.identity, body: step.body },
|
||||
options: step.options
|
||||
}
|
||||
]
|
||||
}
|
||||
})
|
||||
} finally {
|
||||
await current.close()
|
||||
}
|
||||
const reopened = await JournalSqlDatabase.open(test.root, options)
|
||||
try {
|
||||
expect(
|
||||
await reopened.readers.domainHistory({
|
||||
sessionId: 'new-current-chat',
|
||||
direction: 'after',
|
||||
limit: 10
|
||||
})
|
||||
).toMatchObject({
|
||||
page: { items: [{ body: { blocks: [{ text: 'saved after legacy retirement' }] } }] }
|
||||
})
|
||||
expect(existsSync(test.filename)).toBe(false)
|
||||
} finally {
|
||||
await reopened.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('retains downgrade writes, a rollback journal and a nonempty WAL without changing their bytes', async () => {
|
||||
for (const kind of ['append', 'journal', 'wal'] as const) {
|
||||
const test = await profile()
|
||||
await test.retirement.run()
|
||||
test.arm()
|
||||
if (kind === 'append') {
|
||||
const old = new Database(test.filename)
|
||||
old
|
||||
.prepare('INSERT INTO journal_sessions VALUES(?,?,?)')
|
||||
.run('old-new-chat', 'folder', 'new-epoch')
|
||||
const savedItem = journalSqlRetryRows(true).find((row) => row.kind === 'item')!
|
||||
insertJournalRow(old, 'unopened', {
|
||||
...savedItem,
|
||||
seq: journalSqlRetryRows(true).length + 1,
|
||||
itemId: 'downgraded-text',
|
||||
body: {
|
||||
kind: 'message',
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text: 'saved by the downgraded build' }]
|
||||
}
|
||||
})
|
||||
insertJournalRow(old, 'old-new-chat', {
|
||||
...journalSqlRetryRows(true)[0]!,
|
||||
epoch: 'new-epoch',
|
||||
seq: 1
|
||||
})
|
||||
old.close()
|
||||
} else {
|
||||
writeFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'owed recovery bytes')
|
||||
}
|
||||
const before = readFileSync(test.filename)
|
||||
await test.retirement.run()
|
||||
expect(existsSync(test.filename)).toBe(true)
|
||||
expect(readFileSync(test.filename)).toEqual(before)
|
||||
if (kind === 'append') {
|
||||
expect(
|
||||
test
|
||||
.source()
|
||||
.prepare("SELECT session_id FROM journal_sessions WHERE session_id='old-new-chat'")
|
||||
.get()
|
||||
).toBeDefined()
|
||||
expect(
|
||||
test
|
||||
.source()
|
||||
.prepare(
|
||||
"SELECT row_json FROM journal_rows WHERE session_id='unopened' ORDER BY seq DESC LIMIT 1"
|
||||
)
|
||||
.get()!.row_json
|
||||
).toContain('saved by the downgraded build')
|
||||
} else {
|
||||
expect(readFileSync(test.filename + (kind === 'wal' ? '-wal' : '-journal'), 'utf8')).toBe(
|
||||
'owed recovery bytes'
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('does not earn grace from forward wall-clock steps and re-stamps a future completion', () => {
|
||||
const state = {
|
||||
version: 1 as const,
|
||||
proof: 'proof',
|
||||
proofId: 'id',
|
||||
completedAt: 100,
|
||||
process: 'prior',
|
||||
credit: 0,
|
||||
creditRevision: 0,
|
||||
lastWall: 100,
|
||||
phase: 'grace' as const
|
||||
}
|
||||
expect(advanceLegacyRetirementClock(state, 691200100, 3600000, 'next').credit).toBe(3600000)
|
||||
expect(
|
||||
advanceLegacyRetirementClock(
|
||||
{ ...state, completedAt: 9000, credit: 604800000 },
|
||||
1000,
|
||||
10,
|
||||
'next'
|
||||
)
|
||||
).toMatchObject({ completedAt: 1000, credit: 0, process: 'next' })
|
||||
|
||||
let mono = 0n
|
||||
vi.spyOn(process.hrtime, 'bigint').mockImplementation(() => mono)
|
||||
const clock = new JournalLegacyGraceClock()
|
||||
clock.persisted(state)
|
||||
mono = 3600000000000n
|
||||
expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000)
|
||||
clock.pause()
|
||||
expect(clock.observe(state, 691200100, 'next').credit).toBe(3600000)
|
||||
const ambiguousCommit = { ...state, credit: 3600000, creditRevision: 1 }
|
||||
mono = 7200000000000n
|
||||
expect(clock.observe(ambiguousCommit, 691200100, 'next').credit).toBe(3600000)
|
||||
})
|
||||
|
||||
it('retains queued-only text, recovery obligations and orphan history', async () => {
|
||||
for (const kind of ['queue', 'repair', 'orphan', 'extra-profile-column'] as const) {
|
||||
const test = await profile((legacy) => {
|
||||
if (kind === 'queue') {
|
||||
legacy.exec(
|
||||
'DELETE FROM journal_rows; INSERT INTO queued_messages(session_id,message_id,position,body_json,fingerprint,created_at,host_instance,state,hold_reason) VALUES(\'unopened\',\'held\',1,\'{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}\',\'f\',1,\'h\',\'waiting\',\'manual\')'
|
||||
)
|
||||
} else if (kind === 'repair') {
|
||||
legacy.exec("INSERT INTO journal_repairs VALUES('unopened','epoch',1,100)")
|
||||
} else if (kind === 'extra-profile-column') {
|
||||
legacy.exec(
|
||||
"ALTER TABLE agent_session_records ADD COLUMN future_text TEXT; INSERT INTO agent_session_records VALUES('unopened','{}','saved future field')"
|
||||
)
|
||||
} else {
|
||||
legacy.exec("INSERT INTO journal_rows VALUES('orphan','old',1,1,'orphan user text')")
|
||||
}
|
||||
})
|
||||
if (kind === 'queue') {
|
||||
const current = await JournalSqlDatabase.open(test.root, {
|
||||
writerPath: journalSqlWorkerPath(),
|
||||
readerPath: journalSqlReaderPath(),
|
||||
onMaintenanceError: () => {}
|
||||
})
|
||||
try {
|
||||
await current.openSession({
|
||||
sessionId: 'unopened',
|
||||
workspaceId: 'folder-without-git',
|
||||
hostId: 'synthetic-host',
|
||||
agent: 'codex',
|
||||
providerHandle: null
|
||||
})
|
||||
} finally {
|
||||
await current.close()
|
||||
}
|
||||
}
|
||||
const before = readFileSync(test.filename)
|
||||
await test.retirement.run()
|
||||
expect(test.errors.length).toBeGreaterThan(0)
|
||||
expect(readFileSync(test.filename)).toEqual(before)
|
||||
if (kind === 'queue') {
|
||||
expect(test.source().prepare('SELECT body_json FROM queued_messages').get()).toEqual({
|
||||
body_json:
|
||||
'{"kind":"message","role":"user","blocks":[{"type":"text","text":"held user text"}]}'
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('resumes only the prepared empty companions and preserves a recreated main', async () => {
|
||||
for (const recreate of [false, true]) {
|
||||
const test = await profile()
|
||||
await test.retirement.run()
|
||||
await test.recycle()
|
||||
const shm = `${test.filename}-shm`
|
||||
writeFileSync(shm, 'derived index')
|
||||
const state = {
|
||||
...test.receiptState(),
|
||||
phase: 'prepared',
|
||||
manifest: {
|
||||
'-shm': journalLegacyFileIdentity(shm),
|
||||
...(existsSync(`${test.filename}-wal`)
|
||||
? { '-wal': journalLegacyFileIdentity(`${test.filename}-wal`) }
|
||||
: {})
|
||||
}
|
||||
}
|
||||
test.saveState(state)
|
||||
unlinkSync(test.filename)
|
||||
if (recreate) {
|
||||
writeFileSync(test.filename, 'new old-build history')
|
||||
}
|
||||
await test.retirement.run()
|
||||
if (recreate) {
|
||||
expect(readFileSync(shm, 'utf8')).toBe('derived index')
|
||||
}
|
||||
if (recreate) {
|
||||
expect(readFileSync(test.filename, 'utf8')).toBe('new old-build history')
|
||||
} else {
|
||||
expect(test.errors).toEqual([])
|
||||
expect(existsSync(shm)).toBe(false)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,303 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { lstat, unlink } from 'node:fs/promises'
|
||||
import { ownsOrcadProfileDirectory } from '../../orcad/instance-lock/orcad-instance-lock-hold'
|
||||
import { journalDatabasePath } from './database/journal-host-database'
|
||||
import { journalDomainDatabasePath } from './database/journal-domain-database'
|
||||
import {
|
||||
LEGACY_COMPANION_SUFFIXES as SUFFIXES,
|
||||
legacyFilePresent as filePresent,
|
||||
syncLegacyDirectory,
|
||||
removeLegacyCompanions,
|
||||
syncLegacyDestination,
|
||||
legacyDestinationIdentity,
|
||||
type LegacyCompanionManifest as Manifest,
|
||||
type LegacyDestinationIdentity
|
||||
} from './journal-legacy-retirement-files'
|
||||
import {
|
||||
LEGACY_RETIREMENT_WEEK,
|
||||
decodeRetirement,
|
||||
JournalLegacyGraceClock,
|
||||
type Retirement
|
||||
} from './journal-legacy-retirement-state'
|
||||
import {
|
||||
captureJournalLegacySourceReceipt,
|
||||
journalLegacyFileIdentity
|
||||
} from './database/journal-legacy-source-receipt'
|
||||
import { acquireJournalLegacyFileLock } from './journal-legacy-file-lock'
|
||||
import type { JournalSqlOwner } from './journal-sql-owner'
|
||||
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
|
||||
import type { JournalSqlConversions } from './journal-sql-conversions'
|
||||
import { completeJournalLegacyProof } from './journal-legacy-source-proof'
|
||||
|
||||
const WEEK = LEGACY_RETIREMENT_WEEK
|
||||
const SAVE_INTERVAL = 6 * 60 * 60 * 1000
|
||||
const PROCESS_TOKEN = randomUUID()
|
||||
|
||||
/** Existing maintenance owns proof, grace and one fixed unlink per yield; failures only retain data. */
|
||||
export class JournalLegacyRetirement {
|
||||
private readonly filename: string
|
||||
private readonly clock = new JournalLegacyGraceClock()
|
||||
private lastSaved = 0
|
||||
private diagnosed?: string
|
||||
private stopped = new AbortController()
|
||||
|
||||
constructor(
|
||||
private readonly directory: string,
|
||||
private readonly writer: Pick<JournalSqlOwner, 'maintain'>,
|
||||
private readonly readers: Pick<
|
||||
JournalSqlReaderPool,
|
||||
'legacyProof' | 'importState' | 'recycleSourceReaders' | 'releaseSourceReaders'
|
||||
>,
|
||||
private readonly conversions: Pick<JournalSqlConversions, 'ensure'>,
|
||||
private readonly boundary: () => Promise<boolean>,
|
||||
private readonly report: (error: unknown) => void,
|
||||
private readonly ownsProfile = () => ownsOrcadProfileDirectory(directory),
|
||||
private readonly lock = acquireJournalLegacyFileLock
|
||||
) {
|
||||
this.filename = journalDatabasePath(directory)
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
this.clock.pause()
|
||||
this.stopped.abort()
|
||||
}
|
||||
}
|
||||
|
||||
async run(): Promise<void> {
|
||||
try {
|
||||
await this.attempt()
|
||||
} catch (error) {
|
||||
const reason = error instanceof Error ? error.message : String(error)
|
||||
if (this.diagnosed !== reason) {
|
||||
this.diagnosed = reason
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async flushCredit(): Promise<void> {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
const saved = await this.savedState()
|
||||
const state = decodeRetirement(saved.retirement)
|
||||
if (
|
||||
!state ||
|
||||
state.phase !== 'grace' ||
|
||||
state.proof !== saved.receipt ||
|
||||
!this.matches(state.proof)
|
||||
) {
|
||||
return
|
||||
}
|
||||
const advanced = this.clock.observe(state, Date.now(), PROCESS_TOKEN)
|
||||
if (advanced.completedAt === state.completedAt && advanced.credit > state.credit) {
|
||||
await this.save(advanced, false)
|
||||
}
|
||||
} catch (error) {
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
|
||||
private async savedState() {
|
||||
const saved = await this.writer.maintain({ kind: 'legacy-retirement-state' })
|
||||
if (
|
||||
!saved ||
|
||||
typeof saved !== 'object' ||
|
||||
!('kind' in saved) ||
|
||||
saved.kind !== 'legacy-retirement-state'
|
||||
) {
|
||||
throw new Error('legacy retirement state is unavailable')
|
||||
}
|
||||
return saved
|
||||
}
|
||||
|
||||
private async save(
|
||||
state: Retirement,
|
||||
durable: boolean,
|
||||
destination?: LegacyDestinationIdentity
|
||||
): Promise<void> {
|
||||
state.creditRevision++
|
||||
if (
|
||||
(await this.writer.maintain({
|
||||
kind: 'legacy-retirement-save',
|
||||
value: JSON.stringify(state),
|
||||
durable,
|
||||
destination
|
||||
})) !== true
|
||||
) {
|
||||
throw new Error('legacy retirement commit was not acknowledged')
|
||||
}
|
||||
this.lastSaved = Date.now()
|
||||
this.clock.persisted(state)
|
||||
}
|
||||
|
||||
private matches(proof: string): boolean {
|
||||
return JSON.stringify(captureJournalLegacySourceReceipt(this.filename)) === proof
|
||||
}
|
||||
|
||||
private prove(): Promise<void> {
|
||||
return completeJournalLegacyProof(
|
||||
this.readers,
|
||||
this.conversions,
|
||||
this.writer,
|
||||
this.boundary,
|
||||
this.stopped.signal
|
||||
)
|
||||
}
|
||||
|
||||
private async attempt(): Promise<void> {
|
||||
if (this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
const saved = await this.savedState()
|
||||
if (!saved.receipt) {
|
||||
return
|
||||
}
|
||||
const receipt = JSON.parse(saved.receipt)
|
||||
if (
|
||||
receipt.version !== 1 ||
|
||||
receipt.profileCopyInventoryVersion !== 1 ||
|
||||
receipt.disposition !== 'copied-source' ||
|
||||
receipt.wal?.kind !== 'empty'
|
||||
) {
|
||||
return
|
||||
}
|
||||
const proof = saved.receipt
|
||||
let state = decodeRetirement(saved.retirement)
|
||||
if (state && state.proof !== saved.receipt) {
|
||||
throw new Error('legacy retirement provenance changed')
|
||||
}
|
||||
if (state?.phase === 'complete') {
|
||||
if (!(await filePresent(this.filename))) {
|
||||
return
|
||||
}
|
||||
state = { ...state, phase: 'prepared' }
|
||||
}
|
||||
if (state?.phase === 'prepared' && !(await filePresent(this.filename))) {
|
||||
await this.removeCompanions(state)
|
||||
return
|
||||
}
|
||||
if (!this.matches(proof)) {
|
||||
throw new Error('legacy source changed; retaining the original')
|
||||
}
|
||||
const now = Date.now()
|
||||
if (!state) {
|
||||
await this.prove()
|
||||
if (!this.matches(proof)) {
|
||||
throw new Error('legacy source changed during proof')
|
||||
}
|
||||
state = {
|
||||
version: 1,
|
||||
proof: saved.receipt,
|
||||
proofId: randomUUID(),
|
||||
completedAt: now,
|
||||
process: PROCESS_TOKEN,
|
||||
credit: 0,
|
||||
creditRevision: 0,
|
||||
lastWall: now,
|
||||
phase: 'grace'
|
||||
}
|
||||
await this.save(state, true)
|
||||
return
|
||||
}
|
||||
const advanced = this.clock.observe(state, now, PROCESS_TOKEN)
|
||||
if (advanced.process !== state.process || advanced.completedAt !== state.completedAt) {
|
||||
await this.prove()
|
||||
if (!this.matches(proof)) {
|
||||
return
|
||||
}
|
||||
await this.save(advanced, true)
|
||||
return
|
||||
}
|
||||
state = advanced
|
||||
if (state.credit < WEEK || state.process === PROCESS_TOKEN) {
|
||||
if (now - this.lastSaved >= SAVE_INTERVAL) {
|
||||
await this.save(state, false)
|
||||
}
|
||||
return
|
||||
}
|
||||
await this.prove()
|
||||
if (!this.ownsProfile() || !this.matches(proof)) {
|
||||
return
|
||||
}
|
||||
const recycled = await this.readers.recycleSourceReaders()
|
||||
if (!recycled) {
|
||||
return
|
||||
}
|
||||
const signal = AbortSignal.any([recycled.signal, this.stopped.signal])
|
||||
let lease: Awaited<ReturnType<typeof acquireJournalLegacyFileLock>> | undefined
|
||||
try {
|
||||
lease = await this.lock(this.filename, signal)
|
||||
const validate = () => {
|
||||
if (!lease?.held() || signal.aborted || !this.ownsProfile() || !this.matches(proof)) {
|
||||
throw new Error('legacy retirement lost exclusive ownership')
|
||||
}
|
||||
}
|
||||
validate()
|
||||
if (await filePresent(`${this.filename}-journal`)) {
|
||||
throw new Error('legacy rollback journal retained')
|
||||
}
|
||||
const manifest: Manifest = {}
|
||||
for (const suffix of SUFFIXES) {
|
||||
if (await filePresent(this.filename + suffix)) {
|
||||
const file = await lstat(this.filename + suffix)
|
||||
if (!file.isFile()) {
|
||||
throw new Error('legacy companion is not a regular file')
|
||||
}
|
||||
manifest[suffix] = journalLegacyFileIdentity(this.filename + suffix)
|
||||
}
|
||||
}
|
||||
if (manifest['-wal'] && manifest['-wal'].size !== '0') {
|
||||
throw new Error('legacy WAL still contains data')
|
||||
}
|
||||
const domain = journalDomainDatabasePath(this.directory)
|
||||
const destination = legacyDestinationIdentity(domain)
|
||||
await syncLegacyDestination(this.directory, this.filename)
|
||||
if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) {
|
||||
throw new Error('journal destination changed during sync')
|
||||
}
|
||||
state = { ...state, phase: 'prepared', manifest }
|
||||
await this.save(state, true, destination)
|
||||
validate()
|
||||
if (JSON.stringify(legacyDestinationIdentity(domain)) !== JSON.stringify(destination)) {
|
||||
throw new Error('journal destination changed after prepare')
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
await lease.close()
|
||||
lease = undefined
|
||||
}
|
||||
if (!this.ownsProfile() || signal.aborted || !this.matches(proof)) {
|
||||
return
|
||||
}
|
||||
await unlink(this.filename)
|
||||
await syncLegacyDirectory(this.directory)
|
||||
await lease?.close()
|
||||
lease = undefined
|
||||
this.readers.releaseSourceReaders(recycled)
|
||||
await this.removeCompanions(state)
|
||||
} finally {
|
||||
try {
|
||||
await lease?.close()
|
||||
} finally {
|
||||
this.readers.releaseSourceReaders(recycled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async removeCompanions(state: Retirement): Promise<void> {
|
||||
if (
|
||||
await removeLegacyCompanions(
|
||||
this.directory,
|
||||
this.filename,
|
||||
state.manifest,
|
||||
this.boundary,
|
||||
this.ownsProfile,
|
||||
this.stopped.signal
|
||||
)
|
||||
) {
|
||||
await this.save({ ...state, phase: 'complete' }, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
import { readStoredQueuedMessageRow } from './queued-messages/queued-message-stored-row'
|
||||
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
|
||||
import type { JournalSqlConversions } from './journal-sql-conversions'
|
||||
import type { JournalSqlOwner } from './journal-sql-owner'
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
|
||||
const PROFILE_TABLES = new Set([
|
||||
'agent_session_records',
|
||||
'agent_session_operations',
|
||||
'agent_session_retired_claim_keys',
|
||||
'agent_session_tabs',
|
||||
'agent_session_store_meta',
|
||||
'agent_session_command_receipts',
|
||||
'agent_session_attachment_claims',
|
||||
'agent_session_attachment_sweeps',
|
||||
'pr_chat_actions',
|
||||
'pr_check_events',
|
||||
'pr_event_deliveries',
|
||||
'pr_pending_heads',
|
||||
'pr_pending_actions',
|
||||
'pr_check_contexts',
|
||||
'sqlite_sequence'
|
||||
])
|
||||
const CONTENT_TABLES = [
|
||||
'journal_rows',
|
||||
'journal_outcome_generations',
|
||||
'journal_outcome_epochs',
|
||||
'journal_outcomes',
|
||||
'queued_messages'
|
||||
] as const
|
||||
const SOURCE_COLUMNS: Readonly<Record<string, string>> = {
|
||||
journal_sessions: 'session_id workspace_id epoch',
|
||||
journal_rows: 'session_id epoch seq ts row_json',
|
||||
journal_outcome_generations: 'generation session_id epoch',
|
||||
journal_outcome_epochs: 'session_id epoch generation format through_seq prefix_seq',
|
||||
journal_outcomes: 'generation seq field entry_key format ordinal value_json deleted',
|
||||
queued_messages:
|
||||
'session_id message_id position body_json fingerprint created_at host_instance state hold_reason returned_reason returned_rejection settled_at settled_by_op consumed_as carried_from queued_epoch queued_sequence options_json'
|
||||
}
|
||||
const RECOVERY_TABLES = ['journal_repairs', 'journal_imports', 'journal_set_aside']
|
||||
export type LegacyProofCursor = { table: number; after: string | null }
|
||||
export type LegacySourceSession = {
|
||||
sessionId: string
|
||||
workspaceId: string
|
||||
epoch: string
|
||||
empty: boolean
|
||||
}
|
||||
export type LegacyProofPage = { next: LegacyProofCursor | null; sessions: LegacySourceSession[] }
|
||||
|
||||
/** Every saved content row must belong to the current catalog before any source file can retire. */
|
||||
export function readJournalLegacyProofPage(
|
||||
db: Database.Database,
|
||||
cursor: LegacyProofCursor,
|
||||
destination: Database.Database
|
||||
): LegacyProofPage {
|
||||
const tables = new Set(
|
||||
db
|
||||
.prepare("SELECT name FROM sqlite_schema WHERE type='table'")
|
||||
.all()
|
||||
.map((row) => String(row.name))
|
||||
)
|
||||
if (cursor.table === 0 && cursor.after === null) {
|
||||
const version = Number(db.pragma('user_version', { simple: true }))
|
||||
if (version !== 3 && version !== 4) {
|
||||
throw new Error('unsupported legacy source inventory')
|
||||
}
|
||||
for (const table of tables) {
|
||||
const names =
|
||||
PROFILE_TABLES.has(table) && table !== 'sqlite_sequence'
|
||||
? destination
|
||||
.prepare(`PRAGMA table_info(${table})`)
|
||||
.all()
|
||||
.map((column) => column.name)
|
||||
: SOURCE_COLUMNS[table]?.split(' ')
|
||||
if (names) {
|
||||
const columns = new Set(names)
|
||||
if (
|
||||
db
|
||||
.prepare(`PRAGMA table_info(${table})`)
|
||||
.all()
|
||||
.some((column) => !columns.has(column.name))
|
||||
) {
|
||||
throw new Error('legacy source has unaccounted columns')
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
PROFILE_TABLES.has(table) ||
|
||||
CONTENT_TABLES.includes(table as (typeof CONTENT_TABLES)[number]) ||
|
||||
table === 'journal_sessions'
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const quoted = `"${table.replaceAll('"', '""')}"`
|
||||
if (db.prepare(`SELECT 1 FROM ${quoted} LIMIT 1`).get()) {
|
||||
throw new Error('legacy source has unaccounted recovery data')
|
||||
}
|
||||
if (!RECOVERY_TABLES.includes(table)) {
|
||||
throw new Error('legacy source has an unknown table')
|
||||
}
|
||||
}
|
||||
}
|
||||
if (cursor.table < CONTENT_TABLES.length) {
|
||||
const table = CONTENT_TABLES[cursor.table]!
|
||||
if (!tables.has(table)) {
|
||||
return { next: { table: cursor.table + 1, after: null }, sessions: [] }
|
||||
}
|
||||
const columns =
|
||||
table === 'journal_outcomes'
|
||||
? 'generation'
|
||||
: table === 'queued_messages'
|
||||
? `session_id,${
|
||||
db
|
||||
.prepare('PRAGMA table_info(queued_messages)')
|
||||
.all()
|
||||
.some((column) => column.name === 'queued_epoch')
|
||||
? 'queued_epoch'
|
||||
: 'NULL'
|
||||
} AS epoch`
|
||||
: table === 'journal_rows'
|
||||
? 'session_id,epoch'
|
||||
: 'session_id,epoch,generation'
|
||||
const statement = db.prepare(
|
||||
`SELECT rowid AS rowid,${columns} FROM ${table} ${cursor.after === null ? '' : 'WHERE rowid>?'} ORDER BY rowid LIMIT 128`
|
||||
)
|
||||
statement.setReadBigInts(true)
|
||||
const rows = statement.all(...(cursor.after === null ? [] : [BigInt(cursor.after)]))
|
||||
for (const row of rows) {
|
||||
if (
|
||||
table === 'queued_messages' &&
|
||||
!readStoredQueuedMessageRow(
|
||||
db.prepare('SELECT * FROM queued_messages WHERE rowid=?').get(row.rowid!)
|
||||
)
|
||||
) {
|
||||
throw new Error('legacy source has unreadable queued text')
|
||||
}
|
||||
|
||||
const session =
|
||||
table === 'journal_outcomes'
|
||||
? db
|
||||
.prepare(
|
||||
'SELECT session_id,epoch FROM journal_outcome_generations WHERE generation=?'
|
||||
)
|
||||
.get(row.generation!)
|
||||
: row
|
||||
if (
|
||||
!session ||
|
||||
!db
|
||||
.prepare(
|
||||
`SELECT 1 FROM journal_sessions WHERE session_id=?${
|
||||
table === 'queued_messages' ? '' : ' AND epoch=?'
|
||||
}`
|
||||
)
|
||||
.get(
|
||||
session.session_id!,
|
||||
...(table === 'queued_messages' && session.epoch == null ? [] : [session.epoch!])
|
||||
)
|
||||
) {
|
||||
throw new Error('legacy source has orphan content')
|
||||
}
|
||||
if (table.startsWith('journal_outcome')) {
|
||||
const generation = row.generation
|
||||
if (
|
||||
!db
|
||||
.prepare(
|
||||
'SELECT 1 FROM journal_outcome_epochs WHERE session_id=? AND epoch=? AND generation=?'
|
||||
)
|
||||
.get(session.session_id!, session.epoch!, generation!)
|
||||
) {
|
||||
throw new Error('legacy source has an orphan outcome generation')
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
next: rows.length
|
||||
? { table: cursor.table, after: String(rows.at(-1)!.rowid) }
|
||||
: { table: cursor.table + 1, after: null },
|
||||
sessions: []
|
||||
}
|
||||
}
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT session_id,workspace_id,epoch FROM journal_sessions ${cursor.after === null ? '' : 'WHERE session_id>?'} ORDER BY session_id LIMIT 32`
|
||||
)
|
||||
.all(...(cursor.after === null ? [] : [cursor.after]))
|
||||
const sessions = rows.map((row) => {
|
||||
const id = String(row.session_id)
|
||||
const hasRows = !!db.prepare('SELECT 1 FROM journal_rows WHERE session_id=? LIMIT 1').get(id)
|
||||
const hasQueue =
|
||||
tables.has('queued_messages') &&
|
||||
!!db.prepare('SELECT 1 FROM queued_messages WHERE session_id=? LIMIT 1').get(id)
|
||||
if (!hasRows && hasQueue) {
|
||||
throw new Error('legacy queued-only history still requires its source')
|
||||
}
|
||||
const empty =
|
||||
!hasRows &&
|
||||
!hasQueue &&
|
||||
(!tables.has('journal_outcome_epochs') ||
|
||||
!db.prepare('SELECT 1 FROM journal_outcome_epochs WHERE session_id=? LIMIT 1').get(id))
|
||||
return { sessionId: id, workspaceId: String(row.workspace_id), epoch: String(row.epoch), empty }
|
||||
})
|
||||
return {
|
||||
next: rows.length ? { table: cursor.table, after: String(rows.at(-1)!.session_id) } : null,
|
||||
sessions
|
||||
}
|
||||
}
|
||||
|
||||
export async function completeJournalLegacyProof(
|
||||
readers: Pick<JournalSqlReaderPool, 'legacyProof' | 'importState'>,
|
||||
conversions: Pick<JournalSqlConversions, 'ensure'>,
|
||||
writer: Pick<JournalSqlOwner, 'maintain'>,
|
||||
boundary: () => Promise<boolean>,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
let cursor: LegacyProofCursor | null = { table: 0, after: null }
|
||||
while (cursor && !signal.aborted) {
|
||||
if (!(await boundary())) {
|
||||
throw new Error('legacy proof was interrupted')
|
||||
}
|
||||
const page = await readers.legacyProof(cursor)
|
||||
for (const source of page.sessions) {
|
||||
await conversions.ensure(source.sessionId, false)
|
||||
if ((await readers.importState(source.sessionId)).state !== 'ready' && source.empty) {
|
||||
await writer.maintain({
|
||||
kind: 'import-finish',
|
||||
header: {
|
||||
sessionId: source.sessionId,
|
||||
workspaceId: source.workspaceId,
|
||||
epoch: source.epoch,
|
||||
head: {
|
||||
lastSequence: 0,
|
||||
highestFence: 0,
|
||||
contextBoundary: 0,
|
||||
lastActivityAt: 0,
|
||||
derivedTurnItemId: null,
|
||||
historyOrigin: null,
|
||||
queueFailureBoundary: 0,
|
||||
queuePauseMarks: { latestStop: null, resumedSequence: 0, reopenedSequence: 0 }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
if ((await readers.importState(source.sessionId)).state !== 'ready') {
|
||||
throw new Error('legacy source still has unconverted or queued history')
|
||||
}
|
||||
}
|
||||
cursor = page.next
|
||||
}
|
||||
if (signal.aborted) {
|
||||
throw new Error('legacy proof was interrupted')
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import { JournalLegacyRetirement } from './journal-legacy-retirement'
|
||||
import { setImmediate, setTimeout as delay } from 'node:timers/promises'
|
||||
import type { JournalSqlOwner } from './journal-sql-owner'
|
||||
import type { JournalSqlReaderPool } from './journal-sql-reader-pool'
|
||||
@@ -30,6 +31,19 @@ export class JournalSqlBackgroundMaintenance {
|
||||
private readonly onError: (error: unknown, sessionId?: string) => void
|
||||
) {}
|
||||
|
||||
private legacyRetirement?: JournalLegacyRetirement
|
||||
|
||||
ownLegacySource(directory: string, readers: JournalSqlReaderPool): void {
|
||||
this.legacyRetirement = new JournalLegacyRetirement(
|
||||
directory,
|
||||
this.writer,
|
||||
readers,
|
||||
this.conversions,
|
||||
() => this.boundary(),
|
||||
this.onError
|
||||
)
|
||||
}
|
||||
|
||||
start(): Promise<void> {
|
||||
if (this.stopped.signal.aborted) {
|
||||
return Promise.resolve()
|
||||
@@ -75,13 +89,15 @@ export class JournalSqlBackgroundMaintenance {
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.legacyRetirement?.stop()
|
||||
this.stopped.abort()
|
||||
clearTimeout(this.timer)
|
||||
this.timer = undefined
|
||||
}
|
||||
|
||||
settled(): Promise<void> {
|
||||
return this.running ?? Promise.resolve()
|
||||
async settled(): Promise<void> {
|
||||
await this.running
|
||||
await this.legacyRetirement?.flushCredit()
|
||||
}
|
||||
|
||||
private async boundary(): Promise<boolean> {
|
||||
@@ -102,6 +118,7 @@ export class JournalSqlBackgroundMaintenance {
|
||||
private async run(): Promise<void> {
|
||||
while (!this.stopped.signal.aborted && (this.discover || this.sweep || this.periodic)) {
|
||||
await this.runOrdinaryWork()
|
||||
await this.legacyRetirement?.run()
|
||||
if (this.discover) {
|
||||
this.discover = false
|
||||
try {
|
||||
|
||||
@@ -37,7 +37,7 @@ export class JournalSqlBatch {
|
||||
async run<Command, Value>(
|
||||
commands: readonly Command[],
|
||||
apply: (command: Command) => Value | Promise<Value>,
|
||||
durability: 'normal' | 'full' = 'normal'
|
||||
durability: 'normal' | 'full' | 'retirement' = 'normal'
|
||||
): Promise<JournalSqlBatchResult<Value>> {
|
||||
if (this.failure) {
|
||||
throw new JournalSqlBatchError('not-dispatched', this.failure)
|
||||
@@ -47,13 +47,24 @@ export class JournalSqlBatch {
|
||||
return { outcomes }
|
||||
}
|
||||
const drain = commands.slice()
|
||||
const pragmas =
|
||||
durability === 'retirement'
|
||||
? ['synchronous', 'busy_timeout', 'fullfsync', 'checkpoint_fullfsync'].map(
|
||||
(name) => [name, this.db.pragma(name, { simple: true })] as const
|
||||
)
|
||||
: []
|
||||
let began = false
|
||||
let committing = false
|
||||
let failed: JournalSqlBatchError | undefined
|
||||
try {
|
||||
await this.checkpoint?.beforeWrite()
|
||||
if (durability === 'full') {
|
||||
if (durability !== 'normal') {
|
||||
this.db.pragma('synchronous=FULL')
|
||||
if (durability === 'retirement') {
|
||||
this.db.pragma('busy_timeout=0')
|
||||
this.db.pragma('fullfsync=ON')
|
||||
this.db.pragma('checkpoint_fullfsync=ON')
|
||||
}
|
||||
}
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
began = true
|
||||
@@ -89,9 +100,15 @@ export class JournalSqlBatch {
|
||||
this.failure = failed
|
||||
}
|
||||
} finally {
|
||||
if (durability === 'full' && !this.db.isTransaction) {
|
||||
if (durability !== 'normal' && !this.db.isTransaction) {
|
||||
try {
|
||||
this.db.pragma('synchronous=NORMAL')
|
||||
if (durability === 'retirement') {
|
||||
for (const [name, value] of pragmas) {
|
||||
this.db.pragma(`${name}=${Number(value)}`)
|
||||
}
|
||||
} else {
|
||||
this.db.pragma('synchronous=NORMAL')
|
||||
}
|
||||
} catch (cause) {
|
||||
failed ??= new JournalSqlBatchError('indeterminate', cause)
|
||||
this.failure = failed
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { LegacyDestinationIdentity } from './journal-legacy-retirement-files'
|
||||
import { rekeyWslDriveUncChats } from '../../wsl/filesystem/wsl-drive-unc-chat-rekey'
|
||||
import type { JournalForkStageOperation, JournalForkStageResult } from './journal-fork-stages'
|
||||
import { saveJournalDomainOutlineCache, type JournalOutlineCache } from './journal-domain-outline'
|
||||
@@ -26,6 +27,13 @@ import { installAgentSessionStoreRows } from '../../runtime/agent-session/record
|
||||
import type { JournalSqlStoredRecords } from './journal-sql-database-read'
|
||||
|
||||
export type JournalSqlMaintenance =
|
||||
| { kind: 'legacy-retirement-state' }
|
||||
| {
|
||||
kind: 'legacy-retirement-save'
|
||||
value: string
|
||||
durable: boolean
|
||||
destination?: LegacyDestinationIdentity
|
||||
}
|
||||
| { kind: 'records-install' }
|
||||
| { kind: 'wsl-workspace-rekey'; worktreeIds: readonly (readonly [string, string])[] }
|
||||
| JournalForkStageOperation
|
||||
@@ -44,6 +52,7 @@ export type JournalSqlMaintenance =
|
||||
export type JournalSqlProjectionState = JournalDomainImportState
|
||||
|
||||
export type JournalSqlMaintenanceResult =
|
||||
| { kind: 'legacy-retirement-state'; receipt: string | null; retirement: string | null }
|
||||
| { kind: 'records'; records: JournalSqlStoredRecords }
|
||||
| JournalForkStageResult
|
||||
| JournalSqlProjectionState
|
||||
@@ -59,6 +68,25 @@ export function applyJournalSqlMaintenance(
|
||||
operation: JournalSqlMaintenance
|
||||
): JournalSqlMaintenanceResult {
|
||||
switch (operation.kind) {
|
||||
case 'legacy-retirement-state': {
|
||||
const value = (key: string) =>
|
||||
db
|
||||
.prepare('SELECT value FROM agent_session_store_meta WHERE key=?')
|
||||
.get(key)
|
||||
?.value?.toString() ?? null
|
||||
return {
|
||||
kind: 'legacy-retirement-state',
|
||||
receipt: value('journal.legacy-source-copy.v1'),
|
||||
retirement: value('journal.legacy-retirement.v1')
|
||||
}
|
||||
}
|
||||
case 'legacy-retirement-save':
|
||||
db.prepare('INSERT OR REPLACE INTO agent_session_store_meta(key,value) VALUES(?,?)').run(
|
||||
'journal.legacy-retirement.v1',
|
||||
operation.value
|
||||
)
|
||||
return true
|
||||
|
||||
case 'wsl-workspace-rekey':
|
||||
return rekeyWslDriveUncChats(db, operation.worktreeIds)
|
||||
case 'records-install': {
|
||||
|
||||
@@ -207,3 +207,21 @@ it('correlates interleaved replies and resumes current bodies after writer closu
|
||||
})
|
||||
expect(reset.page.items).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('serves domain reads while source recycling yields to a new chat', async () => {
|
||||
const { filename, root } = await fixture()
|
||||
const pool = new JournalSqlReaderPool(
|
||||
filename,
|
||||
join(root, 'absent-legacy.db'),
|
||||
journalSqlReaderPath()
|
||||
)
|
||||
readers.push(pool)
|
||||
await pool.importState('session')
|
||||
const lease = await pool.recycleSourceReaders()
|
||||
expect(lease).not.toBeNull()
|
||||
const conversion = pool.prepareConversion('new-current-chat')
|
||||
expect(await pool.importState('session')).toMatchObject({ state: 'ready' })
|
||||
pool.releaseSourceReaders(lease!)
|
||||
expect(await conversion).toBeNull()
|
||||
expect(await pool.importState('session')).toMatchObject({ state: 'ready' })
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { readJournalLegacyProofPage } from './journal-legacy-source-proof'
|
||||
import { JournalForkPreparation } from './journal-fork-preparation'
|
||||
import { parentPort, workerData } from 'node:worker_threads'
|
||||
import { existsSync } from 'node:fs'
|
||||
@@ -83,6 +84,7 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => {
|
||||
case 'database':
|
||||
reply({ type: 'database', id, result: readJournalSqlDatabase(db, operation.operation) })
|
||||
break
|
||||
case 'legacy-proof':
|
||||
case 'conversion-prepare': {
|
||||
// Each frozen-source read is its own short statement; no snapshot spans the restore.
|
||||
if (!original && existsSync(workerData.originalFilename)) {
|
||||
@@ -92,6 +94,17 @@ port.on('message', ({ id, operation }: JournalSqlReaderRequest) => {
|
||||
timeout: 5000
|
||||
})
|
||||
}
|
||||
if (operation.kind === 'legacy-proof') {
|
||||
if (!original) {
|
||||
throw new Error('legacy source disappeared during proof')
|
||||
}
|
||||
reply({
|
||||
type: 'legacy-proof',
|
||||
id,
|
||||
page: readJournalLegacyProofPage(original, operation.cursor, db)
|
||||
})
|
||||
break
|
||||
}
|
||||
const prepared = original ? prepareJournalDomainImport(original, operation.sessionId) : null
|
||||
if (prepared) {
|
||||
const { pages, ...header } = prepared
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { LegacyProofCursor } from './journal-legacy-source-proof'
|
||||
import type { JournalSqlForkReader, JournalSqlConversionReader } from './journal-sql-reader-slot'
|
||||
import { ReaderSlot } from './journal-sql-reader-slot'
|
||||
import type { JournalForkPreparationInput } from './journal-fork-preparation'
|
||||
@@ -22,6 +23,8 @@ export class JournalSqlReaderPool {
|
||||
private closed = false
|
||||
private closing?: Promise<void>
|
||||
private conversionSlot?: ReaderSlot
|
||||
private sourceLease?: AbortController
|
||||
private sourceRelease?: PromiseWithResolvers<void>
|
||||
private forkSlot?: ReaderSlot
|
||||
private forkRelease?: PromiseWithResolvers<void>
|
||||
|
||||
@@ -99,12 +102,61 @@ export class JournalSqlReaderPool {
|
||||
}
|
||||
|
||||
async prepareConversion(sessionId: string): Promise<JournalSqlConversionReader | null> {
|
||||
await this.releaseSourceLease()
|
||||
if (!this.conversionSlot || this.conversionSlot.failed) {
|
||||
this.conversionSlot = this.slot()
|
||||
}
|
||||
return this.conversionSlot.prepareConversion(sessionId)
|
||||
}
|
||||
|
||||
async legacyProof(cursor: LegacyProofCursor) {
|
||||
await this.releaseSourceLease()
|
||||
this.conversionSlot ??= this.slot()
|
||||
const response = await this.conversionSlot.request({ kind: 'legacy-proof', cursor })
|
||||
if (response.type !== 'legacy-proof') {
|
||||
throw new Error('legacy proof reader returned no page')
|
||||
}
|
||||
return response.page
|
||||
}
|
||||
|
||||
async recycleSourceReaders(): Promise<AbortController | null> {
|
||||
if (
|
||||
this.closed ||
|
||||
this.sourceLease ||
|
||||
this.forkSlot ||
|
||||
this.slots.some((slot) => slot.pendingCount !== 0)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const lease = new AbortController()
|
||||
this.sourceLease = lease
|
||||
this.sourceRelease = Promise.withResolvers<void>()
|
||||
const retired = this.slots.splice(0)
|
||||
this.conversionSlot = undefined
|
||||
try {
|
||||
await Promise.all([...this.retiring, ...retired.map((slot) => slot.close())])
|
||||
return lease
|
||||
} catch (error) {
|
||||
this.releaseSourceReaders(lease)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
releaseSourceReaders(lease: AbortController): void {
|
||||
if (this.sourceLease === lease) {
|
||||
this.sourceLease = undefined
|
||||
this.sourceRelease?.resolve()
|
||||
this.sourceRelease = undefined
|
||||
}
|
||||
}
|
||||
|
||||
private async releaseSourceLease(): Promise<void> {
|
||||
if (this.sourceLease) {
|
||||
this.sourceLease.abort()
|
||||
await this.sourceRelease?.promise
|
||||
}
|
||||
}
|
||||
|
||||
async importState(sessionId: string) {
|
||||
const response = await this.slot().request({ kind: 'import-state', sessionId })
|
||||
if (response.type !== 'import-state') {
|
||||
@@ -159,6 +211,7 @@ export class JournalSqlReaderPool {
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
this.sourceLease?.abort()
|
||||
this.closed = true
|
||||
this.forkRelease?.resolve()
|
||||
return (this.closing ??= Promise.all([
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { LegacyProofCursor, LegacyProofPage } from './journal-legacy-source-proof'
|
||||
import type {
|
||||
JournalForkPreparationInput,
|
||||
JournalForkPreparationResult
|
||||
@@ -14,6 +15,7 @@ import type { JournalDomainHistoryRequest } from './journal-domain-history-page'
|
||||
import type { JournalDomainImportState } from './journal-domain-import-rows'
|
||||
|
||||
export type JournalSqlReaderOperation =
|
||||
| { kind: 'legacy-proof'; cursor: LegacyProofCursor }
|
||||
| { kind: 'fork-prepare'; input: JournalForkPreparationInput }
|
||||
| { kind: 'fork-next'; preparationId: number }
|
||||
| { kind: 'fork-release'; preparationId: number }
|
||||
@@ -27,6 +29,7 @@ export type JournalSqlReaderOperation =
|
||||
|
||||
export type JournalSqlReaderRequest = { id: number; operation: JournalSqlReaderOperation }
|
||||
export type JournalSqlReaderReply =
|
||||
| { type: 'legacy-proof'; id: number; page: LegacyProofPage }
|
||||
| { type: 'fork-prepared'; id: number }
|
||||
| { type: 'fork-next'; id: number; result: JournalForkPreparationResult }
|
||||
| { type: 'ready' }
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { legacyDestinationMain, legacyDestinationIdentity } from './journal-legacy-retirement-files'
|
||||
import { JournalForkStages } from './journal-fork-stages'
|
||||
import { mkdirSync } from 'node:fs'
|
||||
import { dirname } from 'node:path'
|
||||
@@ -34,6 +35,7 @@ let commands: JournalSqlCommand | undefined
|
||||
let forks: JournalForkStages | undefined
|
||||
let checkpoint: JournalCheckpointOwner | undefined
|
||||
let checkpointClosing: Promise<void> | undefined
|
||||
let destinationMain: string | undefined
|
||||
let readOnly = true
|
||||
let fault: unknown
|
||||
let closing = false
|
||||
@@ -48,6 +50,11 @@ async function open(): Promise<void> {
|
||||
}
|
||||
const opened = openJournalDomainDatabase(workerData.filename)
|
||||
db = opened.db
|
||||
try {
|
||||
destinationMain = legacyDestinationMain(workerData.filename)
|
||||
} catch {
|
||||
/* Cleanup eligibility never gates opening a chat. */
|
||||
}
|
||||
readOnly = opened.readOnly
|
||||
forks = new JournalForkStages(db, stateDirectory)
|
||||
commands = new JournalSqlCommand(db, stateDirectory, forks)
|
||||
@@ -126,6 +133,16 @@ async function drain(): Promise<void> {
|
||||
const checkpoint = forks!.checkpoint()
|
||||
try {
|
||||
if (command.type === 'maintenance') {
|
||||
if (command.operation.kind === 'legacy-retirement-save') {
|
||||
if (
|
||||
destinationMain !== legacyDestinationMain(workerData.filename) ||
|
||||
(command.operation.destination &&
|
||||
JSON.stringify(command.operation.destination) !==
|
||||
JSON.stringify(legacyDestinationIdentity(workerData.filename)))
|
||||
) {
|
||||
throw new Error('journal destination generation changed')
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: 'maintenance',
|
||||
id,
|
||||
@@ -154,10 +171,17 @@ async function drain(): Promise<void> {
|
||||
}
|
||||
},
|
||||
writes.some(
|
||||
(request) => request.command.type === 'write' && request.command.command.forkInstall
|
||||
(request) =>
|
||||
request.command.type === 'maintenance' &&
|
||||
request.command.operation.kind === 'legacy-retirement-save' &&
|
||||
request.command.operation.durable
|
||||
)
|
||||
? 'full'
|
||||
: 'normal'
|
||||
? 'retirement'
|
||||
: writes.some(
|
||||
(request) => request.command.type === 'write' && request.command.command.forkInstall
|
||||
)
|
||||
? 'full'
|
||||
: 'normal'
|
||||
)
|
||||
forks!.acknowledge()
|
||||
// All acknowledged commits precede a fault notice; it never retracts a successful COMMIT.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Holding the lock means re-proving it on every heartbeat, not only at acquisition.
|
||||
import { dirname, relative, isAbsolute } from 'node:path'
|
||||
import { statSync, unlinkSync, utimesSync } from 'node:fs'
|
||||
import { startOrcadLockHeartbeat } from './orcad-lock-clone-machine'
|
||||
import { readBoundedLockFile } from './orcad-instance-lock-reclaim'
|
||||
@@ -18,6 +19,22 @@ export type OrcadInstanceLock = {
|
||||
release(options?: { keepUnexpectedExit?: boolean }): void
|
||||
}
|
||||
|
||||
const ownedProfileLocks = new Map<string, OrcadLockRecord>()
|
||||
|
||||
export function ownsOrcadProfileDirectory(directory: string): boolean {
|
||||
for (const [path, record] of ownedProfileLocks) {
|
||||
const child = relative(dirname(path), directory)
|
||||
if (child.startsWith('..') || isAbsolute(child)) {
|
||||
continue
|
||||
}
|
||||
const current = parseOrcadInstanceLockRecord(readBoundedLockFile(path) ?? '')
|
||||
if (current?.nonce === record.nonce && current.pid === process.pid) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type OrcadLockBeat = 'held' | 'lost' | 'unproven'
|
||||
|
||||
function lockIsAbsent(lockPath: string): boolean {
|
||||
@@ -89,6 +106,7 @@ export function holdOrcadInstanceLock(
|
||||
onExclusivityLost: () => void
|
||||
}
|
||||
): OrcadInstanceLock {
|
||||
ownedProfileLocks.set(lockPath, record)
|
||||
let released = false
|
||||
const stopHeartbeat = startOrcadLockHeartbeat(() => {
|
||||
if (beatOrcadInstanceLock(lockPath, record, publish) === 'lost') {
|
||||
@@ -105,6 +123,9 @@ export function holdOrcadInstanceLock(
|
||||
return
|
||||
}
|
||||
released = true
|
||||
if (ownedProfileLocks.get(lockPath)?.nonce === record.nonce) {
|
||||
ownedProfileLocks.delete(lockPath)
|
||||
}
|
||||
stopHeartbeat()
|
||||
// Re-read ownership before unlinking so a successor’s live runtime cannot be unlocked.
|
||||
const current = parseOrcadInstanceLockRecord(readBoundedLockFile(lockPath) ?? '')
|
||||
|
||||
@@ -75,6 +75,7 @@ export const ORCAD_NODE_PTY_JS_ARTIFACTS = [
|
||||
export function orcadNodePtySlotFiles(target: string): string[] {
|
||||
if (isWindowsTarget(target)) {
|
||||
return [
|
||||
'journal-file-lock.exe',
|
||||
'conpty.node',
|
||||
'conpty_console_list.node',
|
||||
'conpty/conpty.dll',
|
||||
@@ -85,7 +86,9 @@ export function orcadNodePtySlotFiles(target: string): string[] {
|
||||
'winpty-agent.exe'
|
||||
]
|
||||
}
|
||||
return target.startsWith('darwin-') ? ['pty.node', 'spawn-helper'] : ['pty.node']
|
||||
return target.startsWith('darwin-')
|
||||
? ['journal-file-lock', 'pty.node', 'spawn-helper']
|
||||
: ['journal-file-lock', 'pty.node']
|
||||
}
|
||||
|
||||
export function orcadNodePtyNativeArtifacts(target: string): string[] {
|
||||
|
||||
Reference in New Issue
Block a user