mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 03:45:08 +00:00
fix: bound retired journal cleanup work per chat
This commit is contained in:
@@ -1,10 +1,22 @@
|
||||
import { afterEach, expect, it } from 'vitest'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { messageFixture } from './journal-sql-message.test-fixture'
|
||||
import { applyJournalSqlMaintenance, type JournalSqlMaintenance } from './journal-sql-maintenance'
|
||||
import { pruneJournalDomainRetiredPage } from './journal-domain-maintenance'
|
||||
import { readJournalSqlDatabase } from './journal-sql-database-read'
|
||||
import { JournalSqlBackgroundMaintenance } from './journal-sql-background-maintenance'
|
||||
import { JournalSqlActivity } from './journal-sql-activity'
|
||||
import type { SqliteBindings } from '../../sqlite/sqlite-statement'
|
||||
|
||||
const databases: Database.Database[] = []
|
||||
afterEach(() => {
|
||||
const maintainers: JournalSqlBackgroundMaintenance[] = []
|
||||
afterEach(async () => {
|
||||
for (const maintenance of maintainers.splice(0)) {
|
||||
maintenance.stop()
|
||||
await maintenance.settled()
|
||||
}
|
||||
vi.restoreAllMocks()
|
||||
vi.useRealTimers()
|
||||
for (const db of databases.splice(0)) {
|
||||
db.close()
|
||||
}
|
||||
@@ -15,13 +27,20 @@ function fixture() {
|
||||
databases.push(db)
|
||||
db.pragma('auto_vacuum=INCREMENTAL')
|
||||
messageFixture(db)
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS journal_item_subagent_names (
|
||||
session_id TEXT NOT NULL, epoch TEXT NOT NULL, subagent_id TEXT NOT NULL, item_id TEXT NOT NULL,
|
||||
PRIMARY KEY(session_id,epoch,subagent_id,item_id)
|
||||
) WITHOUT ROWID`)
|
||||
return db
|
||||
}
|
||||
|
||||
function maintain(db: Database.Database, operation: JournalSqlMaintenance) {
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const result = applyJournalSqlMaintenance(db, operation)
|
||||
const result =
|
||||
operation.kind === 'domain-retired-prune'
|
||||
? pruneJournalDomainRetiredPage(db, operation)
|
||||
: applyJournalSqlMaintenance(db, operation)
|
||||
db.exec('COMMIT')
|
||||
return result
|
||||
} catch (error) {
|
||||
@@ -45,6 +64,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
|
||||
...prefix
|
||||
)
|
||||
db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,?,'deleted',1)`).run(...prefix)
|
||||
db.prepare(`INSERT INTO journal_item_subagent_names VALUES(?,?,'agent','body')`).run(...prefix)
|
||||
db.prepare(`INSERT INTO journal_item_aliases VALUES(?,?,'provider','body',0,NULL)`).run(
|
||||
...prefix
|
||||
)
|
||||
@@ -66,7 +86,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
|
||||
db.exec(`INSERT INTO agent_session_records VALUES('chat','saved record');
|
||||
INSERT INTO agent_session_tabs VALUES('tab','chat',1)`)
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
applyJournalSqlMaintenance(db, { kind: 'domain-retired-prune' })
|
||||
pruneJournalDomainRetiredPage(db, { sessionId: 'chat', epoch: 'epoch' })
|
||||
applyJournalSqlMaintenance(db, { kind: 'compaction-complete', now: 200 })
|
||||
db.exec('ROLLBACK')
|
||||
expect(
|
||||
@@ -80,7 +100,7 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
|
||||
})
|
||||
let more = true
|
||||
while (more) {
|
||||
const result = maintain(db, { kind: 'domain-retired-prune' })
|
||||
const result = maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
|
||||
if (!result || typeof result !== 'object' || !('more' in result)) {
|
||||
throw new Error('expected page')
|
||||
}
|
||||
@@ -96,6 +116,13 @@ it('prunes proven retired epochs with atomic rollback while preserving incomplet
|
||||
expect(
|
||||
db.prepare("SELECT payload_json FROM journal_send_receipts WHERE session_id='chat'").all()
|
||||
).toEqual([{ payload_json: 'accepted receipt' }])
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT epoch,subagent_id,item_id FROM journal_item_subagent_names WHERE session_id='chat'"
|
||||
)
|
||||
.all()
|
||||
).toEqual([{ epoch: 'epoch', subagent_id: 'agent', item_id: 'body' }])
|
||||
expect(
|
||||
db.prepare("SELECT payload_json FROM journal_submission_attempts WHERE session_id='chat'").all()
|
||||
).toEqual([{ payload_json: 'accepted body' }])
|
||||
@@ -129,3 +156,254 @@ it('returns a free page inside the writer transaction and restores it on rollbac
|
||||
maintain(db, { kind: 'page-reclaim' })
|
||||
expect(pages()).toBeLessThan(before)
|
||||
})
|
||||
|
||||
it('bounds a ready-head pass at its captured high key and skips changed or removed heads', () => {
|
||||
const db = fixture()
|
||||
const insert =
|
||||
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
|
||||
VALUES(?,'epoch',0,0)`)
|
||||
for (let index = 0; index < 20; index++) {
|
||||
insert.run(`session-${String(index).padStart(2, '0')}`)
|
||||
}
|
||||
db.exec(`INSERT INTO journal_item_tombstones VALUES('chat','before','old',1),
|
||||
('chat','epoch','current',1),('session-00','before','old',1)`)
|
||||
const page = readJournalSqlDatabase(db, { kind: 'retired-heads', after: null, through: null })
|
||||
expect(page).toMatchObject({ kind: 'retired-heads', through: 'session-19' })
|
||||
if (page.kind !== 'retired-heads') {
|
||||
throw new Error('expected ready heads')
|
||||
}
|
||||
expect(page.heads.map((head) => head.sessionId)).toEqual([
|
||||
'chat',
|
||||
'session-00',
|
||||
'session-01',
|
||||
'session-02',
|
||||
'session-03',
|
||||
'session-04',
|
||||
'session-05',
|
||||
'session-06',
|
||||
'session-07',
|
||||
'session-08',
|
||||
'session-09',
|
||||
'session-10',
|
||||
'session-11',
|
||||
'session-12',
|
||||
'session-13',
|
||||
'session-14'
|
||||
])
|
||||
expect(db.isTransaction).toBe(false)
|
||||
insert.run('session-20')
|
||||
db.exec(`UPDATE journal_chat_state SET epoch='replacement' WHERE session_id='chat';
|
||||
DELETE FROM journal_chat_state WHERE session_id IN ('session-00','session-19')`)
|
||||
for (const target of page.heads.slice(0, 2)) {
|
||||
expect(maintain(db, { kind: 'domain-retired-prune', ...target })).toEqual({
|
||||
kind: 'domain-retired',
|
||||
deleted: 0,
|
||||
more: false
|
||||
})
|
||||
}
|
||||
expect(
|
||||
db
|
||||
.prepare('SELECT session_id,epoch FROM journal_item_tombstones ORDER BY session_id,epoch')
|
||||
.all()
|
||||
).toEqual([
|
||||
{ session_id: 'chat', epoch: 'before' },
|
||||
{ session_id: 'chat', epoch: 'epoch' },
|
||||
{ session_id: 'session-00', epoch: 'before' }
|
||||
])
|
||||
expect(
|
||||
readJournalSqlDatabase(db, {
|
||||
kind: 'retired-heads',
|
||||
after: 'session-14',
|
||||
through: page.through
|
||||
})
|
||||
).toEqual({
|
||||
kind: 'retired-heads',
|
||||
through: 'session-19',
|
||||
heads: [
|
||||
{ sessionId: 'session-15', epoch: 'epoch' },
|
||||
{ sessionId: 'session-16', epoch: 'epoch' },
|
||||
{ sessionId: 'session-17', epoch: 'epoch' },
|
||||
{ sessionId: 'session-18', epoch: 'epoch' }
|
||||
]
|
||||
})
|
||||
expect(
|
||||
readJournalSqlDatabase(db, {
|
||||
kind: 'retired-heads',
|
||||
after: 'session-18',
|
||||
through: page.through
|
||||
})
|
||||
).toEqual({ kind: 'retired-heads', through: 'session-19', heads: [] })
|
||||
expect(
|
||||
readJournalSqlDatabase(db, { kind: 'retired-heads', after: null, through: null })
|
||||
).toMatchObject({ kind: 'retired-heads', through: 'session-20' })
|
||||
})
|
||||
|
||||
it.each([100, 1_000])(
|
||||
'seeks one selected chat around %i current rows and unrelated heads without scanning them',
|
||||
(size) => {
|
||||
const db = fixture()
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
const head =
|
||||
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
|
||||
VALUES(?,'epoch',0,0)`)
|
||||
const current = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'epoch',?,1)`)
|
||||
for (let index = 0; index < size; index++) {
|
||||
const sessionId = `profile-${index}`
|
||||
head.run(sessionId)
|
||||
current.run(sessionId, 'current')
|
||||
current.run('chat', `current-${index}`)
|
||||
}
|
||||
db.exec(`INSERT INTO journal_item_tombstones VALUES('profile-0','before','keep-other-chat',1)`)
|
||||
const retired = db.prepare(`INSERT INTO journal_item_tombstones VALUES('chat',?,?,1)`)
|
||||
for (let index = 0; index < 25; index++) {
|
||||
retired.run('before', `old-${index}`)
|
||||
retired.run('later', `old-${index}`)
|
||||
}
|
||||
db.exec(`INSERT INTO journal_item_subagent_names VALUES
|
||||
('chat','before','agent-a','old'),('chat','later','agent-b','old'),
|
||||
('chat','later','agent-c','old');
|
||||
INSERT INTO journal_events VALUES('chat','later',1,1,'{}')`)
|
||||
db.exec('COMMIT')
|
||||
|
||||
const prepare = db.prepare.bind(db)
|
||||
const queries = new Map<string, SqliteBindings>()
|
||||
const instrumented = new WeakSet<Database.Statement>()
|
||||
vi.spyOn(db, 'prepare').mockImplementation((sql) => {
|
||||
const statement = prepare(sql)
|
||||
if (!instrumented.has(statement)) {
|
||||
instrumented.add(statement)
|
||||
const run = statement.run.bind(statement)
|
||||
vi.spyOn(statement, 'run').mockImplementation((...parameters: SqliteBindings) => {
|
||||
queries.set(sql, parameters)
|
||||
return run(...parameters)
|
||||
})
|
||||
const get = statement.get.bind(statement)
|
||||
vi.spyOn(statement, 'get').mockImplementation((...parameters: SqliteBindings) => {
|
||||
queries.set(sql, parameters)
|
||||
return get(...parameters)
|
||||
})
|
||||
}
|
||||
return statement
|
||||
})
|
||||
expect(
|
||||
maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
|
||||
).toEqual({ kind: 'domain-retired', deleted: 50, more: true })
|
||||
expect(
|
||||
maintain(db, { kind: 'domain-retired-prune', sessionId: 'chat', epoch: 'epoch' })
|
||||
).toEqual({ kind: 'domain-retired', deleted: 4, more: false })
|
||||
vi.restoreAllMocks()
|
||||
|
||||
// Query plans establish the range contract independently of how many heads were seeded.
|
||||
const plans = [...queries].flatMap(([sql, parameters]) =>
|
||||
prepare(`EXPLAIN QUERY PLAN ${sql}`)
|
||||
.all(...parameters)
|
||||
.map((row) => String(row.detail))
|
||||
)
|
||||
expect(plans).toContain(
|
||||
'SEARCH journal_chat_state USING INDEX sqlite_autoindex_journal_chat_state_1 (session_id=?)'
|
||||
)
|
||||
expect(plans).toContain('SEARCH journal_events USING PRIMARY KEY (session_id=? AND epoch>?)')
|
||||
expect(plans.some((detail) => /^SCAN (journal_|ready|retired)/.test(detail))).toBe(false)
|
||||
expect(
|
||||
plans
|
||||
.filter((detail) => detail.startsWith('SEARCH '))
|
||||
.every(
|
||||
(detail) =>
|
||||
detail.startsWith('SEARCH journal_chat_state ') ||
|
||||
/\(session_id=\? AND epoch[<>=]\?/.test(detail)
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE session_id='chat' AND epoch='epoch'"
|
||||
)
|
||||
.get()
|
||||
).toEqual({ count: size })
|
||||
expect(
|
||||
db
|
||||
.prepare(
|
||||
"SELECT epoch,item_id FROM journal_item_tombstones WHERE session_id='profile-0' ORDER BY epoch"
|
||||
)
|
||||
.all()
|
||||
).toEqual([
|
||||
{ epoch: 'before', item_id: 'keep-other-chat' },
|
||||
{ epoch: 'epoch', item_id: 'current' }
|
||||
])
|
||||
expect(
|
||||
db.prepare("SELECT epoch FROM journal_item_subagent_names WHERE session_id='chat'").all()
|
||||
).toEqual([])
|
||||
}
|
||||
)
|
||||
|
||||
it('finishes a retired pass during activity while an already-started conversion is paused on idle', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] })
|
||||
vi.setSystemTime(100)
|
||||
const db = fixture()
|
||||
const activity = new JournalSqlActivity()
|
||||
const paused = Promise.withResolvers<void>()
|
||||
const passFinished = Promise.withResolvers<void>()
|
||||
let converted = false
|
||||
let awaitingPass = false
|
||||
const errors: unknown[] = []
|
||||
const maintenance = new JournalSqlBackgroundMaintenance(
|
||||
{ maintain: async (operation) => maintain(db, operation) },
|
||||
{
|
||||
database: async (operation) => {
|
||||
if (operation.kind === 'sessions') {
|
||||
return { kind: 'sessions', sessionIds: operation.after === null ? ['unready'] : [] }
|
||||
}
|
||||
const result = readJournalSqlDatabase(db, operation)
|
||||
if (awaitingPass && result.kind === 'retired-heads' && result.heads.length === 0) {
|
||||
passFinished.resolve()
|
||||
}
|
||||
return result
|
||||
}
|
||||
},
|
||||
{
|
||||
ensure: async (_sessionId, foreground) => {
|
||||
if (foreground) {
|
||||
throw new Error('background conversion was promoted')
|
||||
}
|
||||
activity.setSessionActive('streaming', true)
|
||||
paused.resolve()
|
||||
await activity.waitForIdle(new AbortController().signal)
|
||||
converted = true
|
||||
return null
|
||||
}
|
||||
},
|
||||
activity,
|
||||
(error) => errors.push(error)
|
||||
)
|
||||
maintainers.push(maintenance)
|
||||
const started = maintenance.start()
|
||||
await paused.promise
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
const head =
|
||||
db.prepare(`INSERT INTO journal_chat_state(session_id,epoch,last_sequence,highest_fence)
|
||||
VALUES(?,'epoch',0,0)`)
|
||||
const retired = db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'before',?,1)`)
|
||||
for (let chat = 0; chat < 20; chat++) {
|
||||
const sessionId = `retired-${String(chat).padStart(2, '0')}`
|
||||
head.run(sessionId)
|
||||
for (let row = 0; row < 53; row++) {
|
||||
retired.run(sessionId, `old-${row}`)
|
||||
}
|
||||
db.prepare(`INSERT INTO journal_item_tombstones VALUES(?,'epoch','current',1)`).run(sessionId)
|
||||
}
|
||||
db.exec('COMMIT')
|
||||
awaitingPass = true
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await passFinished.promise
|
||||
expect(converted).toBe(false)
|
||||
expect(errors).toEqual([])
|
||||
expect(
|
||||
db.prepare("SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE epoch='before'").get()
|
||||
).toEqual({ count: 0 })
|
||||
expect(
|
||||
db.prepare("SELECT COUNT(*) AS count FROM journal_item_tombstones WHERE epoch='epoch'").get()
|
||||
).toEqual({ count: 20 })
|
||||
activity.setSessionActive('streaming', false)
|
||||
await started
|
||||
expect(converted).toBe(true)
|
||||
})
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type Database from '../../sqlite/sync-database'
|
||||
|
||||
export type JournalDomainMaintenance =
|
||||
| { kind: 'domain-retired-prune' }
|
||||
| { kind: 'domain-retired-prune'; sessionId: string; epoch: string }
|
||||
| { kind: 'page-reclaim' }
|
||||
| { kind: 'compaction-state' }
|
||||
| { kind: 'compaction-complete'; now: number }
|
||||
@@ -14,6 +14,7 @@ export type JournalDomainMaintenanceResult =
|
||||
// Only a ready head naming another epoch proves retirement; an interrupted import has no head.
|
||||
const RETIRED_TABLES = [
|
||||
{ name: 'journal_items', key: 'item_id' },
|
||||
{ name: 'journal_item_subagent_names', key: 'subagent_id,item_id' },
|
||||
{ name: 'journal_item_tombstones', key: 'item_id' },
|
||||
{ name: 'journal_item_aliases', key: 'provider_item_id' },
|
||||
{ name: 'journal_submission_attempts', key: 'message_id,attempt_sequence,attempt_index' },
|
||||
@@ -25,24 +26,26 @@ const RETIRED_TABLES = [
|
||||
] as const
|
||||
|
||||
export function pruneJournalDomainRetiredPage(
|
||||
db: Database.Database
|
||||
db: Database.Database,
|
||||
target: { sessionId: string; epoch: string }
|
||||
): JournalDomainMaintenanceResult {
|
||||
const head = db
|
||||
.prepare('SELECT epoch FROM journal_chat_state WHERE session_id=?')
|
||||
.get(target.sessionId)
|
||||
if (head?.epoch !== target.epoch) {
|
||||
return { kind: 'domain-retired', deleted: 0, more: false }
|
||||
}
|
||||
let remaining = 50
|
||||
for (const { name, key } of RETIRED_TABLES) {
|
||||
const columns = key
|
||||
.split(',')
|
||||
.map((column) => `retired.${column}`)
|
||||
.join(',')
|
||||
// Separate ranges seek past the current epoch rather than scanning its live rows.
|
||||
for (const comparison of ['<', '>']) {
|
||||
const result = db
|
||||
.prepare(`DELETE FROM ${name} WHERE (session_id,epoch,${key}) IN (
|
||||
SELECT retired.session_id,retired.epoch,${columns}
|
||||
FROM journal_chat_state AS ready CROSS JOIN ${name} AS retired
|
||||
WHERE retired.session_id=ready.session_id AND retired.epoch${comparison}ready.epoch
|
||||
SELECT session_id,epoch,${key} FROM ${name}
|
||||
WHERE session_id=? AND epoch${comparison}?
|
||||
LIMIT ?
|
||||
)`)
|
||||
.run(remaining)
|
||||
.run(target.sessionId, target.epoch, remaining)
|
||||
remaining -= Number(result.changes)
|
||||
if (remaining === 0) {
|
||||
return { kind: 'domain-retired', deleted: 50, more: true }
|
||||
|
||||
@@ -8,7 +8,7 @@ import type { JournalSqlMaintenance } from './journal-sql-maintenance'
|
||||
const MINUTE = 60_000
|
||||
const COMPACTION_INTERVAL = 6 * 60 * MINUTE
|
||||
|
||||
/** Conversion and retired epochs wait for idle; periodic pruning shares this owner without starvation. */
|
||||
/** Idle conversion and periodic pruning share one owner; activity never blocks retired cleanup. */
|
||||
export class JournalSqlBackgroundMaintenance {
|
||||
private readonly stopped = new AbortController()
|
||||
private running: Promise<void> | undefined
|
||||
@@ -16,6 +16,9 @@ export class JournalSqlBackgroundMaintenance {
|
||||
private sweep = false
|
||||
private periodic = false
|
||||
private reclaim = 0
|
||||
private retiredAfter: string | null = null
|
||||
private retiredThrough: string | null = null
|
||||
private work = 0
|
||||
private wake = new AbortController()
|
||||
private timer: NodeJS.Timeout | undefined
|
||||
|
||||
@@ -101,20 +104,13 @@ export class JournalSqlBackgroundMaintenance {
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
if (this.sweep) {
|
||||
this.sweep = false
|
||||
try {
|
||||
await this.pages({ kind: 'domain-retired-prune' }, Infinity, true)
|
||||
} catch (error) {
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async runOrdinaryWork(): Promise<void> {
|
||||
if (this.periodic && !this.stopped.signal.aborted) {
|
||||
this.periodic = false
|
||||
this.sweep = true
|
||||
try {
|
||||
const state = await this.writer.maintain({ kind: 'compaction-state' })
|
||||
if (
|
||||
@@ -130,7 +126,6 @@ export class JournalSqlBackgroundMaintenance {
|
||||
await this.pages({ kind: 'event-tail-prune', now })
|
||||
if (!this.stopped.signal.aborted) {
|
||||
await this.writer.maintain({ kind: 'compaction-complete', now: Date.now() })
|
||||
this.sweep = true
|
||||
}
|
||||
}
|
||||
await this.pages({ kind: 'page-reclaim' }, this.reclaim)
|
||||
@@ -138,6 +133,46 @@ export class JournalSqlBackgroundMaintenance {
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
if (this.sweep && !this.stopped.signal.aborted) {
|
||||
this.sweep = false
|
||||
try {
|
||||
await this.retiredPages()
|
||||
} catch (error) {
|
||||
this.report(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async retiredPages(): Promise<void> {
|
||||
while (!this.stopped.signal.aborted) {
|
||||
await setImmediate()
|
||||
if (this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
const started = performance.now()
|
||||
const page = await this.readers.database({
|
||||
kind: 'retired-heads',
|
||||
after: this.retiredAfter,
|
||||
through: this.retiredThrough
|
||||
})
|
||||
await this.rest(started)
|
||||
if (page.kind !== 'retired-heads') {
|
||||
throw new Error('journal reader returned no retired heads')
|
||||
}
|
||||
this.retiredThrough = page.through
|
||||
if (page.heads.length === 0) {
|
||||
this.retiredAfter = null
|
||||
this.retiredThrough = null
|
||||
return
|
||||
}
|
||||
for (const target of page.heads) {
|
||||
await this.pages({ kind: 'domain-retired-prune', ...target })
|
||||
if (this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
this.retiredAfter = target.sessionId
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async pages(
|
||||
@@ -145,46 +180,45 @@ export class JournalSqlBackgroundMaintenance {
|
||||
JournalSqlMaintenance,
|
||||
{ kind: 'event-tail-prune' | 'domain-retired-prune' | 'page-reclaim' }
|
||||
>,
|
||||
limit = Infinity,
|
||||
idle = false
|
||||
limit = Infinity
|
||||
): Promise<void> {
|
||||
let work = 0
|
||||
for (let unit = 0; unit < limit && !this.stopped.signal.aborted; unit++) {
|
||||
if (operation.kind !== 'page-reclaim' && this.reclaim > 0) {
|
||||
await this.pages({ kind: 'page-reclaim' }, 1)
|
||||
}
|
||||
if (idle && !(await this.boundary())) {
|
||||
return
|
||||
}
|
||||
await setImmediate()
|
||||
if (this.stopped.signal.aborted) {
|
||||
return
|
||||
}
|
||||
const started = performance.now()
|
||||
const result = await this.writer.maintain(operation)
|
||||
work += performance.now() - started
|
||||
if (!result || typeof result !== 'object' || !('more' in result)) {
|
||||
throw new Error('journal writer returned no maintenance page')
|
||||
}
|
||||
if (operation.kind === 'page-reclaim') {
|
||||
this.reclaim = result.more ? Math.max(0, this.reclaim - 1) : 0
|
||||
}
|
||||
if (work >= 25) {
|
||||
try {
|
||||
await delay(work * 9, undefined, { signal: this.stopped.signal, ref: false })
|
||||
} catch (error) {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
work = 0
|
||||
}
|
||||
await this.rest(started)
|
||||
if (!result.more) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async rest(started: number): Promise<void> {
|
||||
this.work += performance.now() - started
|
||||
if (this.work >= 25) {
|
||||
try {
|
||||
await delay(this.work * 9, undefined, { signal: this.stopped.signal, ref: false })
|
||||
} catch (error) {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
this.work = 0
|
||||
}
|
||||
}
|
||||
|
||||
private async convertSavedChats(): Promise<void> {
|
||||
let after: string | null = null
|
||||
while (await this.boundary()) {
|
||||
@@ -200,7 +234,7 @@ export class JournalSqlBackgroundMaintenance {
|
||||
return
|
||||
}
|
||||
try {
|
||||
await this.conversions.ensure(sessionId, false)
|
||||
await this.convertSavedChat(sessionId)
|
||||
} catch (error) {
|
||||
this.report(error, sessionId)
|
||||
}
|
||||
@@ -209,6 +243,36 @@ export class JournalSqlBackgroundMaintenance {
|
||||
}
|
||||
}
|
||||
|
||||
private async convertSavedChat(sessionId: string): Promise<void> {
|
||||
let complete = false
|
||||
const conversion = this.conversions.ensure(sessionId, false).then(
|
||||
() => {
|
||||
complete = true
|
||||
},
|
||||
(error) => {
|
||||
complete = true
|
||||
this.report(error, sessionId)
|
||||
}
|
||||
)
|
||||
while (!complete && !this.stopped.signal.aborted) {
|
||||
await this.runOrdinaryWork()
|
||||
const signal = AbortSignal.any([this.stopped.signal, this.wake.signal])
|
||||
if (complete || signal.aborted) {
|
||||
continue
|
||||
}
|
||||
let ready = () => {}
|
||||
const wake = new Promise<void>((resolve) => {
|
||||
ready = resolve
|
||||
signal.addEventListener('abort', ready, { once: true })
|
||||
})
|
||||
try {
|
||||
await Promise.race([conversion, wake])
|
||||
} finally {
|
||||
signal.removeEventListener('abort', ready)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private report(error: unknown, sessionId?: string): void {
|
||||
if (!this.stopped.signal.aborted) {
|
||||
this.onError(error, sessionId)
|
||||
|
||||
@@ -14,6 +14,7 @@ export type JournalSqlDatabaseRead =
|
||||
| { kind: 'receipt'; scope: CommandReceiptScope; operationId: string }
|
||||
| { kind: 'epoch'; sessionId: string }
|
||||
| { kind: 'sessions'; after: string | null }
|
||||
| { kind: 'retired-heads'; after: string | null; through: string | null }
|
||||
| { kind: 'attachment-claims'; after: number }
|
||||
| { kind: 'attachment-sweeps'; after: string | null }
|
||||
|
||||
@@ -22,6 +23,11 @@ export type JournalSqlDatabaseReadResult =
|
||||
| { kind: 'receipt'; receipt: ReturnType<typeof readCommandReceipt> }
|
||||
| { kind: 'epoch'; epoch: string | null }
|
||||
| { kind: 'sessions'; sessionIds: string[] }
|
||||
| {
|
||||
kind: 'retired-heads'
|
||||
heads: { sessionId: string; epoch: string }[]
|
||||
through: string | null
|
||||
}
|
||||
| {
|
||||
kind: 'attachment-claims'
|
||||
claims: { rowid: number; uploadId: string; sessionId: string }[]
|
||||
@@ -73,6 +79,31 @@ function read(
|
||||
.all(operation.after)
|
||||
return { kind: 'sessions', sessionIds: rows.map((row) => String(row.session_id)) }
|
||||
}
|
||||
case 'retired-heads': {
|
||||
const through =
|
||||
operation.through ??
|
||||
db
|
||||
.prepare('SELECT session_id FROM journal_chat_state ORDER BY session_id DESC LIMIT 1')
|
||||
.get()?.session_id
|
||||
if (through === undefined) {
|
||||
return { kind: 'retired-heads', heads: [], through: null }
|
||||
}
|
||||
const rows =
|
||||
operation.after === null
|
||||
? db
|
||||
.prepare(`SELECT session_id,epoch FROM journal_chat_state
|
||||
WHERE session_id<=? ORDER BY session_id LIMIT 16`)
|
||||
.all(through)
|
||||
: db
|
||||
.prepare(`SELECT session_id,epoch FROM journal_chat_state
|
||||
WHERE session_id>? AND session_id<=? ORDER BY session_id LIMIT 16`)
|
||||
.all(operation.after, through)
|
||||
return {
|
||||
kind: 'retired-heads',
|
||||
heads: rows.map((row) => ({ sessionId: String(row.session_id), epoch: String(row.epoch) })),
|
||||
through: String(through)
|
||||
}
|
||||
}
|
||||
case 'attachment-claims':
|
||||
return {
|
||||
kind: 'attachment-claims',
|
||||
|
||||
Reference in New Issue
Block a user