mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Own exact completion eligibility at the SQLite source
This commit is contained in:
@@ -11,6 +11,7 @@ import {
|
||||
} from './orchestration-projection-invalidations'
|
||||
import { projectionNeedsRebuild, acknowledgeProjectionDebt } from './orchestration-projection-debt'
|
||||
import { PresentationDispatchIndex } from './presentation-dispatch-index'
|
||||
import { withPresentationCompletionSource } from '../db/dispatch-context/presentation-completion-source'
|
||||
type Fact = { value: unknown; dependencies: readonly string[]; owners: Set<string> }
|
||||
const sources = new WeakMap<OrchestrationDb, OrchestrationProjectionFacts>()
|
||||
const views = new WeakMap<OrchestrationDb, OrchestrationProjectionFacts>()
|
||||
@@ -24,6 +25,7 @@ export class OrchestrationProjectionFacts {
|
||||
private readonly owned = new Map<string, Set<string>>()
|
||||
private owner: string | undefined
|
||||
private version: unknown
|
||||
private revision: number | undefined
|
||||
private cold = true
|
||||
private previousNow = -Infinity
|
||||
|
||||
@@ -106,32 +108,42 @@ export class OrchestrationProjectionFacts {
|
||||
|
||||
refresh(now: number): void {
|
||||
try {
|
||||
const version = this.db.db.pragma('data_version', { simple: true })
|
||||
const overflow = !this.transient && projectionNeedsRebuild(this.db)
|
||||
const cold = this.cold || version !== this.version || now < this.previousNow || overflow
|
||||
if (cold) {
|
||||
this.reset()
|
||||
}
|
||||
const previousSize = this.candidates.size
|
||||
const dirty = this.transient || cold ? [] : projectionDirtyKeys(this.db)
|
||||
const affected = new Set<string>()
|
||||
for (const key of dirty) {
|
||||
for (const factKey of this.dependencies.get(key) ?? []) {
|
||||
for (const owner of this.facts.get(factKey)?.owners ?? []) {
|
||||
affected.add(owner)
|
||||
withPresentationCompletionSource(this.db.db, now, ({ version, revision }) => {
|
||||
const overflow = !this.transient && projectionNeedsRebuild(this.db)
|
||||
const cold =
|
||||
this.cold ||
|
||||
version !== this.version ||
|
||||
revision !== this.revision ||
|
||||
now < this.previousNow ||
|
||||
overflow
|
||||
if (cold) {
|
||||
this.reset()
|
||||
}
|
||||
const previousSize = this.candidates.size
|
||||
const dirty = this.transient || cold ? [] : projectionDirtyKeys(this.db)
|
||||
const affected = new Set<string>()
|
||||
for (const key of dirty) {
|
||||
for (const factKey of this.dependencies.get(key) ?? []) {
|
||||
for (const owner of this.facts.get(factKey)?.owners ?? []) {
|
||||
affected.add(owner)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const owner of affected) {
|
||||
this.release(owner)
|
||||
}
|
||||
this.candidates.refresh(now, dirty, cold)
|
||||
if (!this.transient && (cold || dirty.length > 0 || this.candidates.size !== previousSize)) {
|
||||
acknowledgeProjectionDebt(this.db, this.candidates.size)
|
||||
}
|
||||
this.cold = false
|
||||
this.version = version
|
||||
this.previousNow = now
|
||||
for (const owner of affected) {
|
||||
this.release(owner)
|
||||
}
|
||||
this.candidates.refresh(now, dirty, cold)
|
||||
if (
|
||||
!this.transient &&
|
||||
(cold || dirty.length > 0 || this.candidates.size !== previousSize)
|
||||
) {
|
||||
acknowledgeProjectionDebt(this.db, this.candidates.size)
|
||||
}
|
||||
this.cold = false
|
||||
this.version = version
|
||||
this.revision = revision
|
||||
this.previousNow = now
|
||||
})
|
||||
} catch (error) {
|
||||
// A retry discards partial indexes before consuming the still-unacknowledged obligation.
|
||||
this.cold = true
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { DispatchPresentationDeadlines } from './dispatch-presentation-deadlines'
|
||||
import { dispatchCompletionEpoch } from '../db/dispatch-context/dispatch-completion-epoch'
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../shared/agent-status/contracts/agent-status-freshness'
|
||||
import type { OrchestrationDb } from '../db/orchestration-db'
|
||||
import type { DispatchContextRow } from '../db/orchestration-records'
|
||||
@@ -81,10 +82,10 @@ export class PresentationDispatchIndex {
|
||||
private put(row: Candidate): void {
|
||||
this.candidates.set(row.id, row)
|
||||
if (!active(row) && row.completed_at) {
|
||||
const completed = Date.parse(
|
||||
row.completed_at.includes('T') ? row.completed_at : `${row.completed_at.replace(' ', 'T')}Z`
|
||||
)
|
||||
this.expiry.set(row.id, completed + AGENT_STATUS_STALE_AFTER_MS)
|
||||
const completed = dispatchCompletionEpoch(row.completed_at)
|
||||
if (completed !== null) {
|
||||
this.expiry.set(row.id, completed + AGENT_STATUS_STALE_AFTER_MS)
|
||||
}
|
||||
}
|
||||
for (const key of this.keys(row)) {
|
||||
const bucket = this.addresses.get(key) ?? new Set()
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Preserve the existing ISO/local and SQLite/UTC completion timestamp contract. */
|
||||
export function dispatchCompletionEpoch(value: unknown): number | null {
|
||||
if (typeof value !== 'string') {
|
||||
return null
|
||||
}
|
||||
const epoch = Date.parse(value.includes('T') ? value : `${value.replace(' ', 'T')}Z`)
|
||||
return Number.isFinite(epoch) ? epoch : null
|
||||
}
|
||||
@@ -1,7 +1,14 @@
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
|
||||
import type { DispatchContextRow } from '../orchestration-records'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { DISPATCH_CONTEXT_COLUMN_LIST } from '../row-column-lists'
|
||||
import {
|
||||
DISPATCH_CONTEXT_COLUMN_LIST,
|
||||
DISPATCH_CONTEXT_COLUMNS,
|
||||
selectColumns
|
||||
} from '../row-column-lists'
|
||||
import { dispatchCompletionEpoch } from './dispatch-completion-epoch'
|
||||
import { withPresentationCompletionSource } from './presentation-completion-source'
|
||||
import { completionTable } from './presentation-completion-schema'
|
||||
function presentationAddressKey(column: string, address: string | null): string {
|
||||
return `${column}:${address}`
|
||||
}
|
||||
@@ -11,12 +18,8 @@ export function isRecentPresentationDispatch(dispatch: DispatchContextRow, now:
|
||||
if (!dispatch.completed_at || dispatch.status === 'pending' || dispatch.status === 'dispatched') {
|
||||
return false
|
||||
}
|
||||
const completedAt = Date.parse(
|
||||
dispatch.completed_at.includes('T')
|
||||
? dispatch.completed_at
|
||||
: `${dispatch.completed_at.replace(' ', 'T')}Z`
|
||||
)
|
||||
return Number.isFinite(completedAt) && now - completedAt <= AGENT_STATUS_STALE_AFTER_MS
|
||||
const completedAt = dispatchCompletionEpoch(dispatch.completed_at)
|
||||
return completedAt !== null && now - completedAt <= AGENT_STATUS_STALE_AFTER_MS
|
||||
}
|
||||
|
||||
export type PresentationDispatchCandidate = DispatchContextRow & { presentation_order: number }
|
||||
@@ -26,15 +29,19 @@ export function readPresentationDispatchCandidates(
|
||||
db: OrchestrationDb,
|
||||
now: number
|
||||
): PresentationDispatchCandidate[] {
|
||||
const rows = db.db
|
||||
.prepare(`SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
|
||||
return withPresentationCompletionSource(db.db, now, () => {
|
||||
const rows = db.db
|
||||
.prepare(`SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
|
||||
FROM dispatch_contexts WHERE status IN ('pending', 'dispatched')
|
||||
UNION ALL SELECT rowid AS presentation_order, ${DISPATCH_CONTEXT_COLUMN_LIST}
|
||||
FROM dispatch_contexts WHERE status NOT IN ('pending', 'dispatched')
|
||||
AND julianday(completed_at) >= julianday(?)`)
|
||||
.all(new Date(now - AGENT_STATUS_STALE_AFTER_MS).toISOString())
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: complete schema-owned dispatch projection plus SQLite rowid.
|
||||
return rows as PresentationDispatchCandidate[]
|
||||
UNION ALL SELECT d.rowid AS presentation_order, ${selectColumns(DISPATCH_CONTEXT_COLUMNS, 'd')}
|
||||
FROM temp.${completionTable} c INDEXED BY orchestration_presentation_completion_epoch
|
||||
CROSS JOIN dispatch_contexts d ON d.id = c.id WHERE c.epoch >= ?`)
|
||||
.all(now - AGENT_STATUS_STALE_AFTER_MS)
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: complete schema-owned dispatch projection plus SQLite rowid.
|
||||
return (rows as PresentationDispatchCandidate[]).sort(
|
||||
(a, b) => a.presentation_order - b.presentation_order
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** Active first, otherwise latest per address; an older eligible completion never resurfaces. */
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
|
||||
|
||||
export const completionTable = 'orchestration_presentation_completions'
|
||||
export const completionState = 'orchestration_presentation_completion_state'
|
||||
|
||||
const operationNow = `(SELECT operation_now FROM ${completionState} WHERE id = 1)`
|
||||
const expire = `DELETE FROM ${completionTable} WHERE epoch < ${operationNow} - ${AGENT_STATUS_STALE_AFTER_MS};`
|
||||
const clock = `UPDATE ${completionState} SET operation_now = orchestration_presentation_now() WHERE id = 1;
|
||||
UPDATE ${completionState} SET recovery = recovery OR operation_now < stamp,
|
||||
stamp = operation_now WHERE id = 1; ${expire}`
|
||||
const replace = `INSERT INTO ${completionTable}(id, epoch)
|
||||
SELECT NEW.id, epoch FROM (SELECT orchestration_completion_epoch(NEW.completed_at) AS epoch)
|
||||
WHERE NEW.status NOT IN ('pending', 'dispatched')
|
||||
AND epoch >= ${operationNow} - ${AGENT_STATUS_STALE_AFTER_MS};`
|
||||
|
||||
export const presentationCompletionSchema = `
|
||||
CREATE TEMP TABLE IF NOT EXISTS ${completionTable}(id TEXT PRIMARY KEY, epoch REAL NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS temp.orchestration_presentation_completion_epoch ON ${completionTable}(epoch, id);
|
||||
CREATE TEMP TABLE IF NOT EXISTS ${completionState}(
|
||||
id INTEGER PRIMARY KEY CHECK(id = 1), ready INTEGER NOT NULL DEFAULT 0,
|
||||
version INTEGER, zone TEXT, stamp REAL NOT NULL DEFAULT 0,
|
||||
recovery INTEGER NOT NULL DEFAULT 0, revision INTEGER NOT NULL DEFAULT 0,
|
||||
operation_now REAL NOT NULL DEFAULT 0);
|
||||
INSERT OR IGNORE INTO ${completionState}(id) VALUES(1);
|
||||
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_insert
|
||||
AFTER INSERT ON main.dispatch_contexts BEGIN ${clock} ${replace} END;
|
||||
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_update
|
||||
AFTER UPDATE OF id, status, completed_at ON main.dispatch_contexts
|
||||
BEGIN ${clock} DELETE FROM ${completionTable} WHERE id = OLD.id; ${replace} END;
|
||||
CREATE TEMP TRIGGER IF NOT EXISTS orchestration_presentation_completion_delete
|
||||
AFTER DELETE ON main.dispatch_contexts
|
||||
BEGIN ${clock} DELETE FROM ${completionTable} WHERE id = OLD.id; END;`
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { OrchestrationDb } from '../orchestration-db'
|
||||
import { readPresentationDispatchCandidates } from './dispatch-presentation'
|
||||
import { withPresentationCompletionSource } from './presentation-completion-source'
|
||||
import { orchestrationProjectionFacts } from '../../crew/orchestration-projection-facts'
|
||||
|
||||
const now = Date.parse('2026-11-09T12:00:00Z')
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
function add(db: OrchestrationDb, id: string, completedAt: string) {
|
||||
db.db
|
||||
.prepare(
|
||||
"INSERT INTO dispatch_contexts(id,task_id,assignee_handle,status,completed_at) VALUES(?,'task',?,'completed',?)"
|
||||
)
|
||||
.run(id, id, completedAt)
|
||||
}
|
||||
|
||||
it('reinstalls a completion source whose first initialization rolled back with its caller', () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
try {
|
||||
db.db.exec('BEGIN; SAVEPOINT caller')
|
||||
add(db, 'rolled-back', '2026-11-09T12:00:00Z')
|
||||
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual([
|
||||
'rolled-back'
|
||||
])
|
||||
db.db.exec('ROLLBACK TO caller; RELEASE caller; ROLLBACK')
|
||||
add(db, 'committed', '2026-11-09 12:00:00')
|
||||
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['committed'])
|
||||
db.db
|
||||
.prepare(
|
||||
"UPDATE dispatch_contexts SET id='renamed',completed_at='2020-01-01T00:00:00Z' WHERE id='committed'"
|
||||
)
|
||||
.run()
|
||||
expect(readPresentationDispatchCandidates(db, now)).toEqual([])
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('rebuilds warm projection membership when the runtime timezone changes local completion meaning', () => {
|
||||
const prior = process.env.TZ
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
vi.spyOn(Date, 'now').mockReturnValue(now)
|
||||
try {
|
||||
process.env.TZ = 'UTC'
|
||||
add(db, 'local-completion', '2026-11-09T04:00:00')
|
||||
expect([...orchestrationProjectionFacts(db, now)!.rows()]).toEqual([])
|
||||
process.env.TZ = 'America/Los_Angeles'
|
||||
expect([...orchestrationProjectionFacts(db, now)!.rows()].map((row) => row.id)).toEqual([
|
||||
'local-completion'
|
||||
])
|
||||
process.env.TZ = 'UTC'
|
||||
expect([...orchestrationProjectionFacts(db, now)!.rows()]).toEqual([])
|
||||
} finally {
|
||||
if (prior === undefined) {
|
||||
delete process.env.TZ
|
||||
} else {
|
||||
process.env.TZ = prior
|
||||
}
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps normalization and candidate retrieval on one snapshot across an ordinary external commit', () => {
|
||||
const folder = mkdtempSync(join(tmpdir(), 'orca-completion-source-'))
|
||||
const path = join(folder, 'orchestration.sqlite')
|
||||
const db = new OrchestrationDb(path)
|
||||
db.db.pragma('journal_mode = WAL')
|
||||
const other = new OrchestrationDb(path)
|
||||
try {
|
||||
add(db, 'first', '2026-11-09T12:00:00Z')
|
||||
withPresentationCompletionSource(db.db, now, () => {
|
||||
add(other, 'external', '2026-11-09T12:00:00Z')
|
||||
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['first'])
|
||||
})
|
||||
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual([
|
||||
'first',
|
||||
'external'
|
||||
])
|
||||
other.db
|
||||
.prepare("UPDATE dispatch_contexts SET completed_at='2020-01-01T00:00:00Z' WHERE id='first'")
|
||||
.run()
|
||||
expect(readPresentationDispatchCandidates(db, now).map((row) => row.id)).toEqual(['external'])
|
||||
} finally {
|
||||
other.close()
|
||||
db.close()
|
||||
rmSync(folder, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,121 @@
|
||||
import type Database from '../../../../sqlite/sync-database'
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../shared/agent-status/contracts/agent-status-freshness'
|
||||
import { dispatchCompletionEpoch } from './dispatch-completion-epoch'
|
||||
import {
|
||||
completionTable,
|
||||
completionState,
|
||||
presentationCompletionSchema
|
||||
} from './presentation-completion-schema'
|
||||
|
||||
type Snapshot = { revision: number; version: number }
|
||||
const sources = new WeakMap<Database.Database, PresentationCompletionSource>()
|
||||
|
||||
/** Normalization belongs to the connection, independently of disposable projection caches. */
|
||||
class PresentationCompletionSource {
|
||||
private installed = false
|
||||
private snapshot: Snapshot | undefined
|
||||
|
||||
constructor(private readonly db: Database.Database) {
|
||||
db.function('orchestration_completion_epoch', dispatchCompletionEpoch)
|
||||
db.function('orchestration_presentation_now', () => Date.now())
|
||||
}
|
||||
|
||||
read<T>(now: number, read: (snapshot: Snapshot) => T): T {
|
||||
if (this.snapshot) {
|
||||
return read(this.snapshot)
|
||||
}
|
||||
const outer = this.db.isTransaction
|
||||
this.db.exec('SAVEPOINT orchestration_presentation_source')
|
||||
try {
|
||||
this.install()
|
||||
// Acquire the main snapshot before observing its data_version, including an empty source.
|
||||
this.db.prepare('SELECT rowid FROM main.dispatch_contexts LIMIT 1').get()
|
||||
const state = this.db
|
||||
.prepare(`SELECT *, (SELECT data_version FROM pragma_data_version) AS current_version
|
||||
FROM temp.${completionState} WHERE id = 1`)
|
||||
.get()
|
||||
if (
|
||||
!state ||
|
||||
typeof state.current_version !== 'number' ||
|
||||
typeof state.revision !== 'number'
|
||||
) {
|
||||
throw new Error('Orchestration presentation source is unavailable')
|
||||
}
|
||||
const zone = `${process.env.TZ ?? ''}|${Intl.DateTimeFormat().resolvedOptions().timeZone}`
|
||||
let revision = state.revision
|
||||
if (
|
||||
state.ready !== 1 ||
|
||||
state.recovery === 1 ||
|
||||
state.version !== state.current_version ||
|
||||
state.zone !== zone ||
|
||||
typeof state.stamp !== 'number' ||
|
||||
now < state.stamp
|
||||
) {
|
||||
this.rebuild(now)
|
||||
revision++
|
||||
this.db
|
||||
.prepare(`UPDATE temp.${completionState} SET ready = 1, version = ?, zone = ?,
|
||||
recovery = 0, revision = ? WHERE id = 1`)
|
||||
.run(state.current_version, zone, revision)
|
||||
}
|
||||
this.db
|
||||
.prepare(`DELETE FROM temp.${completionTable} WHERE epoch < ?`)
|
||||
.run(now - AGENT_STATUS_STALE_AFTER_MS)
|
||||
this.db.prepare(`UPDATE temp.${completionState} SET stamp = ? WHERE id = 1`).run(now)
|
||||
this.snapshot = { revision, version: state.current_version }
|
||||
const result = read(this.snapshot)
|
||||
this.db.exec('RELEASE orchestration_presentation_source')
|
||||
if (!outer) {
|
||||
this.installed = true
|
||||
}
|
||||
return result
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK TO orchestration_presentation_source')
|
||||
this.db.exec('RELEASE orchestration_presentation_source')
|
||||
throw error
|
||||
} finally {
|
||||
this.snapshot = undefined
|
||||
}
|
||||
}
|
||||
|
||||
private install(): void {
|
||||
if (this.installed) {
|
||||
return
|
||||
}
|
||||
// First initialization inside a caller transaction may have rolled back its TEMP schema.
|
||||
if (
|
||||
!this.db
|
||||
.prepare('SELECT 1 FROM sqlite_temp_master WHERE type = ? AND name = ?')
|
||||
.get('table', completionState)
|
||||
) {
|
||||
this.db.exec(presentationCompletionSchema)
|
||||
}
|
||||
}
|
||||
|
||||
private rebuild(now: number): void {
|
||||
this.db.prepare(`DELETE FROM temp.${completionTable}`).run()
|
||||
const insert = this.db.prepare(`INSERT INTO temp.${completionTable}(id, epoch) VALUES (?, ?)`)
|
||||
for (const row of this.db
|
||||
.prepare(`SELECT id, completed_at FROM dispatch_contexts
|
||||
WHERE status NOT IN ('pending', 'dispatched')`)
|
||||
.iterate()) {
|
||||
const epoch = dispatchCompletionEpoch(row.completed_at)
|
||||
if (epoch !== null && epoch >= now - AGENT_STATUS_STALE_AFTER_MS) {
|
||||
insert.run(row.id, epoch)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function withPresentationCompletionSource<T>(
|
||||
db: Database.Database,
|
||||
now: number,
|
||||
read: (snapshot: Snapshot) => T
|
||||
): T {
|
||||
let source = sources.get(db)
|
||||
if (!source) {
|
||||
source = new PresentationCompletionSource(db)
|
||||
sources.set(db, source)
|
||||
}
|
||||
return source.read(now, read)
|
||||
}
|
||||
@@ -91,6 +91,10 @@ class SyncDatabase {
|
||||
this.db.exec(sql)
|
||||
}
|
||||
|
||||
function(name: string, callback: Parameters<DatabaseSync['function']>[1]): void {
|
||||
this.db.function(name, callback)
|
||||
}
|
||||
|
||||
prepare(sql: string): SqliteStatement {
|
||||
const cached = this.statementCache.get(sql)
|
||||
if (cached) {
|
||||
|
||||
Reference in New Issue
Block a user