fix(orchestration): let worker_done settle when SQLite cannot write a temp file (#38358)

On a host whose temp directory is full or unwritable, every worker_done
failed and the task never settled (#38123). The settlement runs inside
savepoints, and SQLite keeps savepoint journals in a temp file by default.
When that file cannot be written, SQLite rolls back the whole transaction
itself, so our own rollback then failed with 'cannot rollback - no
transaction is active' and hid the real error. Heartbeats and questions
use no savepoints, which is why they kept working.

The orchestration database now keeps temp storage in memory, and the
lifecycle rollback skips when SQLite already ended the transaction so any
remaining failure of this kind reports its own error.
This commit is contained in:
Jinwoo Hong
2026-11-05 02:02:23 -05:00
committed by GitHub
parent 7e99398ff5
commit f0e7eaa070
3 changed files with 23 additions and 0 deletions
@@ -1,5 +1,6 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from './orchestration-db'
import { runLifecycleWriteTransaction } from './lifecycle-write-transaction-runner'
describe('guarded lifecycle transitions', () => {
let db: OrchestrationDb | undefined
@@ -41,4 +42,20 @@ describe('guarded lifecycle transitions', () => {
expect(db.getTask(task.id)).toMatchObject({ status: 'ready', result: null })
})
it('reports the failure when SQLite already ended the transaction (#38123)', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ runId: 'run_legacy_local', spec: 'aborted transaction' })
// Stands in for the disk-full and I/O errors that make SQLite roll back the whole transaction.
db.db.exec(`CREATE TEMP TRIGGER abort_task_update BEFORE UPDATE ON tasks
BEGIN SELECT RAISE(ROLLBACK, 'database or disk is full'); END`)
expect(() =>
runLifecycleWriteTransaction(db!.db, 'aborted_test', () =>
db!.transitionLifecycle({ entity: 'task', id: task.id, from: 'ready', to: 'completed' })
)
).toThrow('database or disk is full')
expect(db.db.isTransaction).toBe(false)
expect(db.getTask(task.id)?.status).toBe('ready')
})
})
@@ -32,6 +32,10 @@ export function rollbackLifecycleWriteTransaction(
db: Database.Database,
transaction: LifecycleWriteTransaction
): void {
// SQLite ends the whole transaction itself on I/O and disk-full errors; rolling back again would replace that error.
if (!db.isTransaction) {
return
}
if (transaction.savepoint) {
db.exec(`ROLLBACK TO ${transaction.savepoint}`)
db.exec(`RELEASE ${transaction.savepoint}`)
@@ -40,6 +40,8 @@ class OrchestrationDbCore {
this.db.pragma('journal_mode = WAL')
this.db.pragma('synchronous = NORMAL')
this.db.pragma('busy_timeout = 5000')
// Savepoint journals otherwise need a temp file; a full or unwritable temp dir then aborts the whole transaction (#38123).
this.db.pragma('temp_store = MEMORY')
createTables.call(this as unknown as OrchestrationDb)
// Read before migrate, which drops the capture triggers it may need to rebuild around.
const removedWorkCaptureWasIntact = isRemovedWorkCaptureIntact(this.db)