From f0e7eaa07067e1d4ea640e8b47d102a0a2176aa7 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:02:23 -0500 Subject: [PATCH] fix(orchestration): let worker_done settle when SQLite cannot write a temp file (#38358) On a host whose temp directory is full or unwritable, every worker_done failed and the task never settled (#38123). The settlement runs inside savepoints, and SQLite keeps savepoint journals in a temp file by default. When that file cannot be written, SQLite rolls back the whole transaction itself, so our own rollback then failed with 'cannot rollback - no transaction is active' and hid the real error. Heartbeats and questions use no savepoints, which is why they kept working. The orchestration database now keeps temp storage in memory, and the lifecycle rollback skips when SQLite already ended the transaction so any remaining failure of this kind reports its own error. --- .../db/lifecycle-transition.test.ts | 17 +++++++++++++++++ .../orchestration/db/lifecycle-transition.ts | 4 ++++ .../orchestration/db/orchestration-db.ts | 2 ++ 3 files changed, 23 insertions(+) diff --git a/src/main/runtime/orchestration/db/lifecycle-transition.test.ts b/src/main/runtime/orchestration/db/lifecycle-transition.test.ts index cb8c45f16ab..d1b8dc4bd17 100644 --- a/src/main/runtime/orchestration/db/lifecycle-transition.test.ts +++ b/src/main/runtime/orchestration/db/lifecycle-transition.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from 'vitest' import { OrchestrationDb } from './orchestration-db' +import { runLifecycleWriteTransaction } from './lifecycle-write-transaction-runner' describe('guarded lifecycle transitions', () => { let db: OrchestrationDb | undefined @@ -41,4 +42,20 @@ describe('guarded lifecycle transitions', () => { expect(db.getTask(task.id)).toMatchObject({ status: 'ready', result: null }) }) + + it('reports the failure when SQLite already ended the transaction (#38123)', () => { + db = new OrchestrationDb(':memory:') + const task = db.createTask({ runId: 'run_legacy_local', spec: 'aborted transaction' }) + // Stands in for the disk-full and I/O errors that make SQLite roll back the whole transaction. + db.db.exec(`CREATE TEMP TRIGGER abort_task_update BEFORE UPDATE ON tasks + BEGIN SELECT RAISE(ROLLBACK, 'database or disk is full'); END`) + + expect(() => + runLifecycleWriteTransaction(db!.db, 'aborted_test', () => + db!.transitionLifecycle({ entity: 'task', id: task.id, from: 'ready', to: 'completed' }) + ) + ).toThrow('database or disk is full') + expect(db.db.isTransaction).toBe(false) + expect(db.getTask(task.id)?.status).toBe('ready') + }) }) diff --git a/src/main/runtime/orchestration/db/lifecycle-transition.ts b/src/main/runtime/orchestration/db/lifecycle-transition.ts index 2e8a618262f..5eefd879ecb 100644 --- a/src/main/runtime/orchestration/db/lifecycle-transition.ts +++ b/src/main/runtime/orchestration/db/lifecycle-transition.ts @@ -32,6 +32,10 @@ export function rollbackLifecycleWriteTransaction( db: Database.Database, transaction: LifecycleWriteTransaction ): void { + // SQLite ends the whole transaction itself on I/O and disk-full errors; rolling back again would replace that error. + if (!db.isTransaction) { + return + } if (transaction.savepoint) { db.exec(`ROLLBACK TO ${transaction.savepoint}`) db.exec(`RELEASE ${transaction.savepoint}`) diff --git a/src/main/runtime/orchestration/db/orchestration-db.ts b/src/main/runtime/orchestration/db/orchestration-db.ts index 8aacd47bdd3..82cba889d3b 100644 --- a/src/main/runtime/orchestration/db/orchestration-db.ts +++ b/src/main/runtime/orchestration/db/orchestration-db.ts @@ -40,6 +40,8 @@ class OrchestrationDbCore { this.db.pragma('journal_mode = WAL') this.db.pragma('synchronous = NORMAL') this.db.pragma('busy_timeout = 5000') + // Savepoint journals otherwise need a temp file; a full or unwritable temp dir then aborts the whole transaction (#38123). + this.db.pragma('temp_store = MEMORY') createTables.call(this as unknown as OrchestrationDb) // Read before migrate, which drops the capture triggers it may need to rebuild around. const removedWorkCaptureWasIntact = isRemovedWorkCaptureIntact(this.db)