mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Merge commit '5df83053ab8' into brennanb2025/native-chat-sqlite-host
This commit is contained in:
@@ -16,8 +16,8 @@ import {
|
||||
waitForIdle,
|
||||
waitForWorking
|
||||
} from './native-chat-4a-session'
|
||||
import { chatDatabasePath, hasTable, openReadOnly } from './native-chat-4a-sqlite-observer'
|
||||
import { streamTextObserver } from './native-chat-4a-source-observer'
|
||||
import { chatDatabasePath } from './native-chat-4a-sqlite-observer'
|
||||
import { streamTextObserver, runningToolsObserver } from './native-chat-4a-source-observer'
|
||||
import {
|
||||
replyPrompt,
|
||||
streamPrompt,
|
||||
@@ -38,55 +38,6 @@ export async function competingSend(
|
||||
return { hostMs: answer.hostMs, durableMs: Date.now() - sentAt }
|
||||
}
|
||||
|
||||
// Only post-baseline rows, newest revision per ID: old running journal revisions cannot prove overlap.
|
||||
function runningToolsObserver(file: string, sessionId: string) {
|
||||
const db = openReadOnly(file)
|
||||
try {
|
||||
const domain = hasTable(db, 'journal_chat_state')
|
||||
const position = domain ? 'ordinal' : 'seq'
|
||||
const source = `${domain ? 'journal_items' : 'journal_rows'} WHERE session_id=? AND epoch=
|
||||
(SELECT epoch FROM ${domain ? 'journal_chat_state' : 'journal_sessions'} WHERE session_id=?)`
|
||||
const baseline = db
|
||||
.prepare(
|
||||
`SELECT epoch,${position} AS position FROM ${source} ORDER BY ${position} DESC LIMIT 1`
|
||||
)
|
||||
.get(sessionId, sessionId)
|
||||
if (!baseline || typeof baseline.epoch !== 'string' || typeof baseline.position !== 'number') {
|
||||
throw new Error('no current source boundary; live contention not exercised')
|
||||
}
|
||||
const tail = `FROM ${source} AND ${position}>? ORDER BY ${position} DESC LIMIT 40`
|
||||
const query = db.prepare(
|
||||
domain
|
||||
? `SELECT epoch,item_id AS id,kind,lifecycle_state AS state ${tail}`
|
||||
: `SELECT epoch,json_extract(row_json,'$.itemId') AS id,
|
||||
json_extract(row_json,'$.body.kind') AS kind,json_extract(row_json,'$.body.state') AS state
|
||||
FROM (SELECT epoch,seq,row_json ${tail}) ORDER BY seq DESC`
|
||||
)
|
||||
return {
|
||||
epoch: baseline.epoch,
|
||||
position: baseline.position,
|
||||
read: () => {
|
||||
const seen = new Set<string>()
|
||||
const running: string[] = []
|
||||
for (const item of query.all(sessionId, sessionId, baseline.position)) {
|
||||
if (item.epoch !== baseline.epoch || typeof item.id !== 'string' || seen.has(item.id)) {
|
||||
continue
|
||||
}
|
||||
seen.add(item.id)
|
||||
if (item.kind === 'tool-call' && item.state === 'running') {
|
||||
running.push(item.id)
|
||||
}
|
||||
}
|
||||
return running
|
||||
},
|
||||
close: () => db.close()
|
||||
}
|
||||
} catch (error) {
|
||||
db.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function phaseSample(
|
||||
ctx: RunContext,
|
||||
shape: StreamCase['shape'],
|
||||
|
||||
@@ -213,7 +213,7 @@ export async function streamingRows(ctx: RunContext): Promise<void> {
|
||||
streamBytes: bytes + 1024 ** 2,
|
||||
largeItems,
|
||||
observation:
|
||||
'outside read-only SQLite current text, latest eight rows, every 100 ms; adds reader load',
|
||||
'outside read-only SQLite current text, latest 40 records per source, every 100 ms; adds reader load',
|
||||
concurrent:
|
||||
'send to another chat, large-chat tail read and Stop start together at threshold'
|
||||
}
|
||||
|
||||
@@ -16,12 +16,20 @@ export function prepareRetiredHistory(
|
||||
try {
|
||||
const domain = hasTable(db, 'journal_chat_state')
|
||||
const heads = domain ? 'journal_chat_state' : 'journal_sessions'
|
||||
const targetHead = db.prepare(`SELECT epoch FROM ${heads} WHERE session_id=?`).get(target)
|
||||
if (
|
||||
!db.prepare(`SELECT 1 FROM ${heads} WHERE session_id=? AND epoch IS NOT NULL`).get(target)
|
||||
typeof targetHead?.epoch !== 'string' ||
|
||||
targetHead.epoch === epoch ||
|
||||
source === target ||
|
||||
!Number.isSafeInteger(limit) ||
|
||||
limit <= 0
|
||||
) {
|
||||
throw new Error('retired fixture target has no ready current head')
|
||||
throw new Error(
|
||||
'retired fixture requires an unrelated ready target, retired epoch and positive count'
|
||||
)
|
||||
}
|
||||
db.exec('BEGIN IMMEDIATE')
|
||||
let generation = 0
|
||||
if (domain) {
|
||||
const columns = db
|
||||
.prepare('PRAGMA table_info(journal_items)')
|
||||
@@ -41,20 +49,39 @@ export function prepareRetiredHistory(
|
||||
limit
|
||||
)
|
||||
} else {
|
||||
db.prepare(`INSERT INTO journal_rows(session_id,epoch,seq,ts,row_json)
|
||||
SELECT ?,?,seq,ts,row_json FROM (
|
||||
SELECT seq,ts,row_json,ROW_NUMBER() OVER (
|
||||
PARTITION BY json_extract(row_json,'$.itemId') ORDER BY seq DESC) AS latest
|
||||
FROM journal_rows WHERE session_id=? AND epoch=
|
||||
(SELECT epoch FROM journal_sessions WHERE session_id=?) AND json_extract(row_json,'$.kind')='item'
|
||||
) WHERE latest=1 ORDER BY seq LIMIT ?`).run(target, epoch, source, source, limit)
|
||||
const head = db
|
||||
.prepare(`SELECT head.* FROM journal_outcome_epochs head JOIN journal_sessions live
|
||||
ON live.session_id=head.session_id AND live.epoch=head.epoch WHERE head.session_id=?`)
|
||||
.get(source)
|
||||
if (!head || head.prefix_seq !== 0 || Number(head.format) < 1 || Number(head.format) > 3) {
|
||||
throw new Error('retired fixture requires a supported complete outcome source')
|
||||
}
|
||||
// No current epoch points at this generation: the main arm's existing sweeper owns its cleanup.
|
||||
generation = Number(
|
||||
db
|
||||
.prepare('INSERT INTO journal_outcome_generations(session_id,epoch) VALUES (?,?)')
|
||||
.run(target, epoch).lastInsertRowid
|
||||
)
|
||||
db.prepare(`INSERT INTO journal_outcomes(generation,seq,field,entry_key,format,ordinal,value_json,deleted)
|
||||
SELECT ?,seq,field,entry_key,format,ordinal,value_json,deleted FROM (
|
||||
SELECT *,ROW_NUMBER() OVER (PARTITION BY entry_key ORDER BY seq DESC) AS latest
|
||||
FROM journal_outcomes WHERE generation=? AND seq<=? AND field='items'
|
||||
) WHERE latest=1 AND deleted=0 ORDER BY ordinal,entry_key LIMIT ?`).run(
|
||||
generation,
|
||||
head.generation,
|
||||
head.through_seq,
|
||||
limit
|
||||
)
|
||||
}
|
||||
const table = domain ? 'journal_items' : 'journal_rows'
|
||||
const payload = domain ? 'payload_json' : 'row_json'
|
||||
const result = db
|
||||
.prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(${payload})),0) AS bytes
|
||||
FROM ${table} WHERE session_id=? AND epoch=?`)
|
||||
.get(target, epoch)!
|
||||
.prepare(
|
||||
domain
|
||||
? `SELECT count(*) AS n,COALESCE(sum(octet_length(payload_json)),0) AS bytes
|
||||
FROM journal_items WHERE session_id=? AND epoch=?`
|
||||
: `SELECT count(*) AS n,COALESCE(sum(octet_length(value_json)),0) AS bytes
|
||||
FROM journal_outcomes WHERE generation=?`
|
||||
)
|
||||
.get(...(domain ? [target, epoch] : [generation]))!
|
||||
if (Number(result.n) !== limit) {
|
||||
throw new Error(`retired fixture copied ${String(result.n)} rows, expected ${limit}`)
|
||||
}
|
||||
@@ -73,13 +100,18 @@ export function retiredHistory(file: string, sessionId: string): RetiredHistory
|
||||
const db = openReadOnly(file)
|
||||
try {
|
||||
const domain = hasTable(db, 'journal_chat_state')
|
||||
const table = domain ? 'journal_items' : 'journal_rows'
|
||||
const heads = domain ? 'journal_chat_state' : 'journal_sessions'
|
||||
const payload = domain ? 'payload_json' : 'row_json'
|
||||
const row = db
|
||||
.prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(item.${payload})),0) AS bytes
|
||||
FROM ${table} item JOIN ${heads} head ON item.session_id=head.session_id
|
||||
WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch`)
|
||||
.prepare(
|
||||
domain
|
||||
? `SELECT count(*) AS n,COALESCE(sum(octet_length(item.payload_json)),0) AS bytes
|
||||
FROM journal_items item JOIN journal_chat_state head ON item.session_id=head.session_id
|
||||
WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch`
|
||||
: `SELECT count(*) AS n,COALESCE(sum(octet_length(item.value_json)),0) AS bytes
|
||||
FROM journal_outcomes item JOIN journal_outcome_generations retired ON retired.generation=item.generation
|
||||
LEFT JOIN journal_outcome_epochs head ON head.session_id=retired.session_id
|
||||
AND head.epoch=retired.epoch AND head.generation=retired.generation
|
||||
WHERE retired.session_id=? AND head.session_id IS NULL`
|
||||
)
|
||||
.get(sessionId)!
|
||||
return { rows: Number(row.n), payloadBytes: Number(row.bytes) }
|
||||
} finally {
|
||||
|
||||
@@ -1,6 +1,81 @@
|
||||
// Read-only source sizes and persisted outline keys, queried outside either product arm.
|
||||
import { openReadOnly, hasTable } from './native-chat-4a-sqlite-observer'
|
||||
|
||||
function openItemSource(file: string, sessionId: string) {
|
||||
const db = openReadOnly(file)
|
||||
try {
|
||||
const domain = hasTable(db, 'journal_chat_state')
|
||||
if (!domain && !hasTable(db, 'journal_outcomes')) {
|
||||
throw new Error('unsupported current-item storage; observation unexercised')
|
||||
}
|
||||
const head = domain
|
||||
? `SELECT session_id,epoch,epoch AS source,
|
||||
(SELECT COALESCE(max(ordinal),0) FROM journal_items
|
||||
WHERE session_id=head.session_id AND epoch=head.epoch) AS position
|
||||
FROM journal_chat_state head WHERE session_id=?`
|
||||
: `SELECT head.session_id,head.epoch,head.generation AS source,head.through_seq AS position
|
||||
FROM journal_outcome_epochs head JOIN journal_sessions live
|
||||
ON live.session_id=head.session_id AND live.epoch=head.epoch
|
||||
WHERE head.session_id=? AND head.prefix_seq=0 AND head.format BETWEEN 1 AND 3`
|
||||
const baseline = db.prepare(head).get(sessionId)
|
||||
if (
|
||||
!baseline ||
|
||||
typeof baseline.epoch !== 'string' ||
|
||||
typeof baseline.position !== 'number' ||
|
||||
(typeof baseline.source !== 'string' && typeof baseline.source !== 'number')
|
||||
) {
|
||||
throw new Error('no supported current item source; observation unexercised')
|
||||
}
|
||||
const tail = domain
|
||||
? `SELECT head.epoch,head.source,item.ordinal AS position,item.item_id AS id,
|
||||
'items' AS field,0 AS deleted,item.rowid AS stored_row
|
||||
FROM journal_items item JOIN head ON item.session_id=head.session_id AND item.epoch=head.epoch
|
||||
WHERE item.ordinal>? ORDER BY item.ordinal DESC LIMIT ?`
|
||||
: `SELECT head.epoch,head.source,item.seq AS position,json_extract(item.entry_key,'$') AS id,
|
||||
item.field,item.deleted,item.rowid AS stored_row
|
||||
FROM journal_outcomes item JOIN head ON item.generation=head.source
|
||||
WHERE item.seq>? AND item.seq<=head.position ORDER BY item.seq DESC LIMIT ?`
|
||||
return {
|
||||
db,
|
||||
epoch: baseline.epoch,
|
||||
source: baseline.source,
|
||||
position: baseline.position,
|
||||
// Sort only bounded metadata; fetch bodies after obsolete revisions/deletions are excluded.
|
||||
items: `WITH head AS (${head}),recent AS (${tail}),latest AS (
|
||||
SELECT *,ROW_NUMBER() OVER (PARTITION BY field,id ORDER BY position DESC) AS rank FROM recent)
|
||||
SELECT latest.*,stored.${domain ? 'payload_json' : 'value_json'} AS json FROM latest
|
||||
JOIN ${domain ? 'journal_items' : 'journal_outcomes'} stored ON stored.rowid=latest.stored_row
|
||||
WHERE latest.rank=1 AND latest.field='items' AND typeof(latest.id)='text'
|
||||
AND latest.deleted=0 AND latest.epoch=? AND latest.source=?`,
|
||||
close: () => db.close()
|
||||
}
|
||||
} catch (error) {
|
||||
db.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Current burst identities only; each read releases its snapshot before the caller dispatches. */
|
||||
export function runningToolsObserver(file: string, sessionId: string) {
|
||||
const source = openItemSource(file, sessionId)
|
||||
try {
|
||||
const query = source.db.prepare(`${source.items}
|
||||
AND json_extract(json,'$.body.kind')='tool-call' AND json_extract(json,'$.body.state')='running'`)
|
||||
return {
|
||||
epoch: source.epoch,
|
||||
position: source.position,
|
||||
read: () =>
|
||||
query
|
||||
.all(sessionId, source.position, 40, source.epoch, source.source)
|
||||
.map((item) => String(item.id)),
|
||||
close: source.close
|
||||
}
|
||||
} catch (error) {
|
||||
source.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export type OutlineCacheState = {
|
||||
epoch: string
|
||||
sequence: number
|
||||
@@ -15,7 +90,7 @@ export type OutlineCacheState = {
|
||||
export function outlineCacheState(file: string, sessionId: string): OutlineCacheState | null {
|
||||
const db = openReadOnly(file)
|
||||
try {
|
||||
if (!hasTable(db, 'journal_outline_cache')) {
|
||||
if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) {
|
||||
return null
|
||||
}
|
||||
const row = db
|
||||
@@ -49,6 +124,9 @@ export function outlineCacheState(file: string, sessionId: string): OutlineCache
|
||||
export function outlineCacheContains(file: string, sessionId: string, value: unknown): boolean {
|
||||
const db = openReadOnly(file)
|
||||
try {
|
||||
if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) {
|
||||
return false
|
||||
}
|
||||
return Boolean(
|
||||
db
|
||||
.prepare(`SELECT 1 FROM journal_outline_cache cache JOIN journal_chat_state head
|
||||
@@ -62,23 +140,23 @@ export function outlineCacheContains(file: string, sessionId: string, value: unk
|
||||
}
|
||||
}
|
||||
|
||||
/** Read only the latest eight stored rows; SQLite returns the current ASCII stream's byte count, never its growing body. */
|
||||
/** At most 40 source records, newest revision per key; SQLite returns bytes, never the growing ASCII body. */
|
||||
export function streamTextObserver(file: string, sessionId: string, marker: string) {
|
||||
const db = openReadOnly(file)
|
||||
const source = hasTable(db, 'journal_chat_state')
|
||||
? `SELECT payload_json AS json FROM journal_items WHERE session_id=? AND epoch=
|
||||
(SELECT epoch FROM journal_chat_state WHERE session_id=?) ORDER BY ordinal DESC LIMIT 8`
|
||||
: `SELECT row_json AS json FROM journal_rows WHERE session_id=? AND epoch=
|
||||
(SELECT epoch FROM journal_sessions WHERE session_id=?) ORDER BY seq DESC LIMIT 8`
|
||||
const query =
|
||||
db.prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes,
|
||||
json_extract(json,'$.body.state') AS state FROM (${source})
|
||||
WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? LIMIT 1`)
|
||||
return {
|
||||
read: () => {
|
||||
const row = query.get(sessionId, sessionId, marker, marker)
|
||||
return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' }
|
||||
},
|
||||
close: () => db.close()
|
||||
const source = openItemSource(file, sessionId)
|
||||
try {
|
||||
const query = source.db
|
||||
.prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes,
|
||||
json_extract(json,'$.body.state') AS state FROM (${source.items})
|
||||
WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? ORDER BY position DESC LIMIT 1`)
|
||||
return {
|
||||
read: () => {
|
||||
const row = query.get(sessionId, -1, 40, source.epoch, source.source, marker, marker)
|
||||
return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' }
|
||||
},
|
||||
close: source.close
|
||||
}
|
||||
} catch (error) {
|
||||
source.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user