Own exact completion eligibility at the SQLite source

This commit is contained in:
Brennan Benson
2026-11-09 16:56:42 -08:00
parent 88222a852e
commit 1bfbc4b756
8 changed files with 320 additions and 43 deletions
@@ -11,6 +11,7 @@ import {
} from './orchestration-projection-invalidations'
import { projectionNeedsRebuild, acknowledgeProjectionDebt } from './orchestration-projection-debt'
import { PresentationDispatchIndex } from './presentation-dispatch-index'
import { withPresentationCompletionSource } from '../db/dispatch-context/presentation-completion-source'
type Fact = { value: unknown; dependencies: readonly string[]; owners: Set<string> }
const sources = new WeakMap<OrchestrationDb, OrchestrationProjectionFacts>()
const views = new WeakMap<OrchestrationDb, OrchestrationProjectionFacts>()
@@ -24,6 +25,7 @@ export class OrchestrationProjectionFacts {
private readonly owned = new Map<string, Set<string>>()
private owner: string | undefined
private version: unknown
private revision: number | undefined
private cold = true
private previousNow = -Infinity
@@ -106,32 +108,42 @@ export class OrchestrationProjectionFacts {
refresh(now: number): void {
try {
const version = this.db.db.pragma('data_version', { simple: true })
const overflow = !this.transient && projectionNeedsRebuild(this.db)
const cold = this.cold || version !== this.version || now < this.previousNow || overflow
if (cold) {
this.reset()
}
const previousSize = this.candidates.size
const dirty = this.transient || cold ? [] : projectionDirtyKeys(this.db)
const affected = new Set<string>()
for (const key of dirty) {
for (const factKey of this.dependencies.get(key) ?? []) {
for (const owner of this.facts.get(factKey)?.owners ?? []) {
affected.add(owner)
withPresentationCompletionSource(this.db.db, now, ({ version, revision }) => {
const overflow = !this.transient && projectionNeedsRebuild(this.db)
const cold =
this.cold ||
version !== this.version ||
revision !== this.revision ||
now < this.previousNow ||
overflow
if (cold) {
this.reset()
}
const previousSize = this.candidates.size
const dirty = this.transient || cold ? [] : projectionDirtyKeys(this.db)
const affected = new Set<string>()
for (const key of dirty) {
for (const factKey of this.dependencies.get(key) ?? []) {
for (const owner of this.facts.get(factKey)?.owners ?? []) {
affected.add(owner)
}
}
}
}
for (const owner of affected) {
this.release(owner)
}
this.candidates.refresh(now, dirty, cold)
if (!this.transient && (cold || dirty.length > 0 || this.candidates.size !== previousSize)) {
acknowledgeProjectionDebt(this.db, this.candidates.size)
}
this.cold = false
this.version = version
this.previousNow = now
for (const owner of affected) {
this.release(owner)
}
this.candidates.refresh(now, dirty, cold)
if (
!this.transient &&
(cold || dirty.length > 0 || this.candidates.size !== previousSize)
) {
acknowledgeProjectionDebt(this.db, this.candidates.size)
}
this.cold = false
this.version = version
this.revision = revision
this.previousNow = now
})
} catch (error) {
// A retry discards partial indexes before consuming the still-unacknowledged obligation.
this.cold = true
@@ -1,4 +1,5 @@
import { DispatchPresentationDeadlines } from './dispatch-presentation-deadlines'
import { dispatchCompletionEpoch } from '../db/dispatch-context/dispatch-completion-epoch'
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../shared/agent-status/contracts/agent-status-freshness'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { DispatchContextRow } from '../db/orchestration-records'
@@ -81,10 +82,10 @@ export class PresentationDispatchIndex {
private put(row: Candidate): void {
this.candidates.set(row.id, row)
if (!active(row) && row.completed_at) {
const completed = Date.parse(
row.completed_at.includes('T') ? row.completed_at : `${row.completed_at.replace(' ', 'T')}Z`
)
this.expiry.set(row.id, completed + AGENT_STATUS_STALE_AFTER_MS)
const completed = dispatchCompletionEpoch(row.completed_at)
if (completed !== null) {
this.expiry.set(row.id, completed + AGENT_STATUS_STALE_AFTER_MS)
}
}
for (const key of this.keys(row)) {
const bucket = this.addresses.get(key) ?? new Set()
@@ -0,0 +1,8 @@
/** Preserve the existing ISO/local and SQLite/UTC completion timestamp contract. */
export function dispatchCompletionEpoch(value: unknown): number | null {
if (typeof value !== 'string') {
return null
}
const epoch = Date.parse(value.includes('T') ? value : `${value.replace(' ', 'T')}Z`)
return Number.isFinite(epoch) ? epoch : null
}
@@ -1,7 +1,14 @@
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
import type { DispatchContextRow } from '../orchestration-records'
import type { OrchestrationDb } from '../orchestration-db'
import { DISPATCH_CONTEXT_COLUMN_LIST } from '../row-column-lists'
import {
DISPATCH_CONTEXT_COLUMN_LIST,
DISPATCH_CONTEXT_COLUMNS,
selectColumns
} from '../row-column-lists'
import { dispatchCompletionEpoch } from './dispatch-completion-epoch'
import { withPresentationCompletionSource } from './presentation-completion-source'
import { completionTable } from './presentation-completion-schema'
function presentationAddressKey(column: string, address: string | null): string {
return `${column}:${address}`
}
@@ -11,12 +18,8 @@ export function isRecentPresentationDispatch(dispatch: DispatchContextRow, now:
if (!dispatch.completed_at || dispatch.status === 'pending' || dispatch.status === 'dispatched') {
return false
}
const completedAt = Date.parse(
dispatch.completed_at.includes('T')
? dispatch.completed_at
: `${dispatch.completed_at.replace(' ', 'T')}Z`
)
return Number.isFinite(completedAt) && now - completedAt <= AGENT_STATUS_STALE_AFTER_MS
const completedAt = dispatchCompletionEpoch(dispatch.completed_at)
return completedAt !== null && now - completedAt <= AGENT_STATUS_STALE_AFTER_MS
}
export type PresentationDispatchCandidate = DispatchContextRow & { presentation_order: number }
@@ -26,15 +29,19 @@ export function readPresentationDispatchCandidates(
db: OrchestrationDb,
now: number
): PresentationDispatchCandidate[] {
const rows = db.db
.prepare(`SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
return withPresentationCompletionSource(db.db, now, () => {
const rows = db.db
.prepare(`SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
FROM dispatch_contexts WHERE status IN ('pending', 'dispatched')
UNION ALL SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
FROM dispatch_contexts WHERE status NOT IN ('pending', 'dispatched')
AND julianday(completed_at) >= julianday(?)`)
.all(new Date(now - AGENT_STATUS_STALE_AFTER_MS).toISOString())
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: complete schema-owned dispatch projection plus SQLite rowid.
return rows as PresentationDispatchCandidate[]
UNION ALL SELECT d.rowid AS presentation_order, ${selectColumns(DISPATCH_CONTEXT_COLUMNS, 'd')}
FROM temp.${completionTable} c INDEXED BY orchestration_presentation_completion_epoch
CROSS JOIN dispatch_contexts d ON d.id = c.id WHERE c.epoch >= ?`)
.all(now - AGENT_STATUS_STALE_AFTER_MS)
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: complete schema-owned dispatch projection plus SQLite rowid.
return (rows as PresentationDispatchCandidate[]).sort(
(a, b) => a.presentation_order - b.presentation_order
)
})
}
/** Active first, otherwise latest per address; an older eligible completion never resurfaces. */
@@ -0,0 +1,32 @@
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
export const completionTable = 'orchestration_presentation_completions'
export const completionState = 'orchestration_presentation_completion_state'
const operationNow = `(SELECT operation_now FROM ${completionState} WHERE id = 1)`
const expire = `DELETE FROM ${completionTable} WHERE epoch < ${operationNow} - ${AGENT_STATUS_STALE_AFTER_MS};`
const clock = `UPDATE ${completionState} SET operation_now = orchestration_presentation_now() WHERE id = 1;
UPDATE ${completionState} SET recovery = recovery OR operation_now < stamp,
stamp = operation_now WHERE id = 1; ${expire}`
const replace = `INSERT INTO ${completionTable}(id, epoch)
SELECT NEW.id, epoch FROM (SELECT orchestration_completion_epoch(NEW.completed_at) AS epoch)
WHERE NEW.status NOT IN ('pending', 'dispatched')
AND epoch >= ${operationNow} - ${AGENT_STATUS_STALE_AFTER_MS};`
export const presentationCompletionSchema = `
CREATE TEMP TABLE IF NOT EXISTS ${completionTable}(id TEXT PRIMARY KEY, epoch REAL NOT NULL);
CREATE INDEX IF NOT EXISTS temp.orchestration_presentation_completion_epoch ON ${completionTable}(epoch, id);
CREATE TEMP TABLE IF NOT EXISTS ${completionState}(
id INTEGER PRIMARY KEY CHECK(id = 1), ready INTEGER NOT NULL DEFAULT 0,
version INTEGER, zone TEXT, stamp REAL NOT NULL DEFAULT 0,
recovery INTEGER NOT NULL DEFAULT 0, revision INTEGER NOT NULL DEFAULT 0,
operation_now REAL NOT NULL DEFAULT 0);
INSERT OR IGNORE INTO ${completionState}(id) VALUES(1);
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_insert
AFTER INSERT ON main.dispatch_contexts BEGIN ${clock} ${replace} END;
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_update
AFTER UPDATE OF id, status, completed_at ON main.dispatch_contexts
BEGIN ${clock} DELETE FROM ${completionTable} WHERE id = OLD.id; ${replace} END;
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_delete
AFTER DELETE ON main.dispatch_contexts
BEGIN ${clock} DELETE FROM ${completionTable} WHERE id = OLD.id; END;`
@@ -0,0 +1,92 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { afterEach, expect, it, vi } from 'vitest'
import { OrchestrationDb } from '../orchestration-db'
import { readPresentationDispatchCandidates } from './dispatch-presentation'
import { withPresentationCompletionSource } from './presentation-completion-source'
import { orchestrationProjectionFacts } from '../../crew/orchestration-projection-facts'
const now = Date.parse('2026-11-09T12:00:00Z')
afterEach(() => vi.restoreAllMocks())
function add(db: OrchestrationDb, id: string, completedAt: string) {
db.db
.prepare(
"INSERT INTO dispatch_contexts(id,task_id,assignee_handle,status,completed_at) VALUES(?,'task',?,'completed',?)"
)
.run(id, id, completedAt)
}
it('reinstalls a completion source whose first initialization rolled back with its caller', () => {
const db = new OrchestrationDb(':memory:')
try {
db.db.exec('BEGIN; SAVEPOINT caller')
add(db, 'rolled-back', '2026-11-09T12:00:00Z')
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual([
'rolled-back'
])
db.db.exec('ROLLBACK TO caller; RELEASE caller; ROLLBACK')
add(db, 'committed', '2026-11-09 12:00:00')
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['committed'])
db.db
.prepare(
"UPDATE dispatch_contexts SET id='renamed',completed_at='2020-01-01T00:00:00Z' WHERE id='committed'"
)
.run()
expect(readPresentationDispatchCandidates(db, now)).toEqual([])
} finally {
db.close()
}
})
it('rebuilds warm projection membership when the runtime timezone changes local completion meaning', () => {
const prior = process.env.TZ
const db = new OrchestrationDb(':memory:')
vi.spyOn(Date, 'now').mockReturnValue(now)
try {
process.env.TZ = 'UTC'
add(db, 'local-completion', '2026-11-09T04:00:00')
expect([...orchestrationProjectionFacts(db, now)!.rows()]).toEqual([])
process.env.TZ = 'America/Los_Angeles'
expect([...orchestrationProjectionFacts(db, now)!.rows()].map((row) => row.id)).toEqual([
'local-completion'
])
process.env.TZ = 'UTC'
expect([...orchestrationProjectionFacts(db, now)!.rows()]).toEqual([])
} finally {
if (prior === undefined) {
delete process.env.TZ
} else {
process.env.TZ = prior
}
db.close()
}
})
it('keeps normalization and candidate retrieval on one snapshot across an ordinary external commit', () => {
const folder = mkdtempSync(join(tmpdir(), 'orca-completion-source-'))
const path = join(folder, 'orchestration.sqlite')
const db = new OrchestrationDb(path)
db.db.pragma('journal_mode = WAL')
const other = new OrchestrationDb(path)
try {
add(db, 'first', '2026-11-09T12:00:00Z')
withPresentationCompletionSource(db.db, now, () => {
add(other, 'external', '2026-11-09T12:00:00Z')
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['first'])
})
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual([
'first',
'external'
])
other.db
.prepare("UPDATE dispatch_contexts SET completed_at='2020-01-01T00:00:00Z' WHERE id='first'")
.run()
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['external'])
} finally {
other.close()
db.close()
rmSync(folder, { recursive: true, force: true })
}
})
@@ -0,0 +1,121 @@
import type Database from '../../../../sqlite/sync-database'
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
import { dispatchCompletionEpoch } from './dispatch-completion-epoch'
import {
completionTable,
completionState,
presentationCompletionSchema
} from './presentation-completion-schema'
type Snapshot = { revision: number; version: number }
const sources = new WeakMap<Database.Database, PresentationCompletionSource>()
/** Normalization belongs to the connection, independently of disposable projection caches. */
class PresentationCompletionSource {
private installed = false
private snapshot: Snapshot | undefined
constructor(private readonly db: Database.Database) {
db.function('orchestration_completion_epoch', dispatchCompletionEpoch)
db.function('orchestration_presentation_now', () => Date.now())
}
read<T>(now: number, read: (snapshot: Snapshot) => T): T {
if (this.snapshot) {
return read(this.snapshot)
}
const outer = this.db.isTransaction
this.db.exec('SAVEPOINT orchestration_presentation_source')
try {
this.install()
// Acquire the main snapshot before observing its data_version, including an empty source.
this.db.prepare('SELECT rowid FROM main.dispatch_contexts LIMIT 1').get()
const state = this.db
.prepare(`SELECT *, (SELECT data_version FROM pragma_data_version) AS current_version
FROM temp.${completionState} WHERE id = 1`)
.get()
if (
!state ||
typeof state.current_version !== 'number' ||
typeof state.revision !== 'number'
) {
throw new Error('Orchestration presentation source is unavailable')
}
const zone = `${process.env.TZ ?? ''}|${Intl.DateTimeFormat().resolvedOptions().timeZone}`
let revision = state.revision
if (
state.ready !== 1 ||
state.recovery === 1 ||
state.version !== state.current_version ||
state.zone !== zone ||
typeof state.stamp !== 'number' ||
now < state.stamp
) {
this.rebuild(now)
revision++
this.db
.prepare(`UPDATE temp.${completionState} SET ready = 1, version = ?, zone = ?,
recovery = 0, revision = ? WHERE id = 1`)
.run(state.current_version, zone, revision)
}
this.db
.prepare(`DELETE FROM temp.${completionTable} WHERE epoch < ?`)
.run(now - AGENT_STATUS_STALE_AFTER_MS)
this.db.prepare(`UPDATE temp.${completionState} SET stamp = ? WHERE id = 1`).run(now)
this.snapshot = { revision, version: state.current_version }
const result = read(this.snapshot)
this.db.exec('RELEASE orchestration_presentation_source')
if (!outer) {
this.installed = true
}
return result
} catch (error) {
this.db.exec('ROLLBACK TO orchestration_presentation_source')
this.db.exec('RELEASE orchestration_presentation_source')
throw error
} finally {
this.snapshot = undefined
}
}
private install(): void {
if (this.installed) {
return
}
// First initialization inside a caller transaction may have rolled back its TEMP schema.
if (
!this.db
.prepare('SELECT 1 FROM sqlite_temp_master WHERE type = ? AND name = ?')
.get('table', completionState)
) {
this.db.exec(presentationCompletionSchema)
}
}
private rebuild(now: number): void {
this.db.prepare(`DELETE FROM temp.${completionTable}`).run()
const insert = this.db.prepare(`INSERT INTO temp.${completionTable}(id, epoch) VALUES (?, ?)`)
for (const row of this.db
.prepare(`SELECT id, completed_at FROM dispatch_contexts
WHERE status NOT IN ('pending', 'dispatched')`)
.iterate()) {
const epoch = dispatchCompletionEpoch(row.completed_at)
if (epoch !== null && epoch >= now - AGENT_STATUS_STALE_AFTER_MS) {
insert.run(row.id, epoch)
}
}
}
}
export function withPresentationCompletionSource<T>(
db: Database.Database,
now: number,
read: (snapshot: Snapshot) => T
): T {
let source = sources.get(db)
if (!source) {
source = new PresentationCompletionSource(db)
sources.set(db, source)
}
return source.read(now, read)
}
+4
View File
@@ -91,6 +91,10 @@ class SyncDatabase {
this.db.exec(sql)
}
function(name: string, callback: Parameters<DatabaseSync['function']>[1]): void {
this.db.function(name, callback)
}
prepare(sql: string): SqliteStatement {
const cached = this.statementCache.get(sql)
if (cached) {