mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(orchestration): let worker_done settle when SQLite cannot write a temp file (#38358)
On a host whose temp directory is full or unwritable, every worker_done failed and the task never settled (#38123). The settlement runs inside savepoints, and SQLite keeps savepoint journals in a temp file by default. When that file cannot be written, SQLite rolls back the whole transaction itself, so our own rollback then failed with 'cannot rollback - no transaction is active' and hid the real error. Heartbeats and questions use no savepoints, which is why they kept working. The orchestration database now keeps temp storage in memory, and the lifecycle rollback skips when SQLite already ended the transaction so any remaining failure of this kind reports its own error.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from './orchestration-db'
|
||||
import { runLifecycleWriteTransaction } from './lifecycle-write-transaction-runner'
|
||||
|
||||
describe('guarded lifecycle transitions', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
@@ -41,4 +42,20 @@ describe('guarded lifecycle transitions', () => {
|
||||
|
||||
expect(db.getTask(task.id)).toMatchObject({ status: 'ready', result: null })
|
||||
})
|
||||
|
||||
it('reports the failure when SQLite already ended the transaction (#38123)', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ runId: 'run_legacy_local', spec: 'aborted transaction' })
|
||||
// Stands in for the disk-full and I/O errors that make SQLite roll back the whole transaction.
|
||||
db.db.exec(`CREATE TEMP TRIGGER abort_task_update BEFORE UPDATE ON tasks
|
||||
BEGIN SELECT RAISE(ROLLBACK, 'database or disk is full'); END`)
|
||||
|
||||
expect(() =>
|
||||
runLifecycleWriteTransaction(db!.db, 'aborted_test', () =>
|
||||
db!.transitionLifecycle({ entity: 'task', id: task.id, from: 'ready', to: 'completed' })
|
||||
)
|
||||
).toThrow('database or disk is full')
|
||||
expect(db.db.isTransaction).toBe(false)
|
||||
expect(db.getTask(task.id)?.status).toBe('ready')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -32,6 +32,10 @@ export function rollbackLifecycleWriteTransaction(
|
||||
db: Database.Database,
|
||||
transaction: LifecycleWriteTransaction
|
||||
): void {
|
||||
// SQLite ends the whole transaction itself on I/O and disk-full errors; rolling back again would replace that error.
|
||||
if (!db.isTransaction) {
|
||||
return
|
||||
}
|
||||
if (transaction.savepoint) {
|
||||
db.exec(`ROLLBACK TO ${transaction.savepoint}`)
|
||||
db.exec(`RELEASE ${transaction.savepoint}`)
|
||||
|
||||
@@ -40,6 +40,8 @@ class OrchestrationDbCore {
|
||||
this.db.pragma('journal_mode = WAL')
|
||||
this.db.pragma('synchronous = NORMAL')
|
||||
this.db.pragma('busy_timeout = 5000')
|
||||
// Savepoint journals otherwise need a temp file; a full or unwritable temp dir then aborts the whole transaction (#38123).
|
||||
this.db.pragma('temp_store = MEMORY')
|
||||
createTables.call(this as unknown as OrchestrationDb)
|
||||
// Read before migrate, which drops the capture triggers it may need to rebuild around.
|
||||
const removedWorkCaptureWasIntact = isRemovedWorkCaptureIntact(this.db)
|
||||
|
||||
Reference in New Issue
Block a user