Merge commit '5df83053ab8' into brennanb2025/native-chat-sqlite-host

This commit is contained in:
Brennan Benson
2026-11-10 11:34:13 -08:00
4 changed files with 151 additions and 90 deletions
@@ -16,8 +16,8 @@ import {
waitForIdle,
waitForWorking
} from './native-chat-4a-session'
import { chatDatabasePath, hasTable, openReadOnly } from './native-chat-4a-sqlite-observer'
import { streamTextObserver } from './native-chat-4a-source-observer'
import { chatDatabasePath } from './native-chat-4a-sqlite-observer'
import { streamTextObserver, runningToolsObserver } from './native-chat-4a-source-observer'
import {
replyPrompt,
streamPrompt,
@@ -38,55 +38,6 @@ export async function competingSend(
return { hostMs: answer.hostMs, durableMs: Date.now() - sentAt }
}
// Only post-baseline rows, newest revision per ID: old running journal revisions cannot prove overlap.
function runningToolsObserver(file: string, sessionId: string) {
const db = openReadOnly(file)
try {
const domain = hasTable(db, 'journal_chat_state')
const position = domain ? 'ordinal' : 'seq'
const source = `${domain ? 'journal_items' : 'journal_rows'} WHERE session_id=? AND epoch=
(SELECT epoch FROM ${domain ? 'journal_chat_state' : 'journal_sessions'} WHERE session_id=?)`
const baseline = db
.prepare(
`SELECT epoch,${position} AS position FROM ${source} ORDER BY ${position} DESC LIMIT 1`
)
.get(sessionId, sessionId)
if (!baseline || typeof baseline.epoch !== 'string' || typeof baseline.position !== 'number') {
throw new Error('no current source boundary; live contention not exercised')
}
const tail = `FROM ${source} AND ${position}>? ORDER BY ${position} DESC LIMIT 40`
const query = db.prepare(
domain
? `SELECT epoch,item_id AS id,kind,lifecycle_state AS state ${tail}`
: `SELECT epoch,json_extract(row_json,'$.itemId') AS id,
json_extract(row_json,'$.body.kind') AS kind,json_extract(row_json,'$.body.state') AS state
FROM (SELECT epoch,seq,row_json ${tail}) ORDER BY seq DESC`
)
return {
epoch: baseline.epoch,
position: baseline.position,
read: () => {
const seen = new Set<string>()
const running: string[] = []
for (const item of query.all(sessionId, sessionId, baseline.position)) {
if (item.epoch !== baseline.epoch || typeof item.id !== 'string' || seen.has(item.id)) {
continue
}
seen.add(item.id)
if (item.kind === 'tool-call' && item.state === 'running') {
running.push(item.id)
}
}
return running
},
close: () => db.close()
}
} catch (error) {
db.close()
throw error
}
}
export async function phaseSample(
ctx: RunContext,
shape: StreamCase['shape'],
@@ -213,7 +213,7 @@ export async function streamingRows(ctx: RunContext): Promise<void> {
streamBytes: bytes + 1024 ** 2,
largeItems,
observation:
'outside read-only SQLite current text, latest eight rows, every 100 ms; adds reader load',
'outside read-only SQLite current text, latest 40 records per source, every 100 ms; adds reader load',
concurrent:
'send to another chat, large-chat tail read and Stop start together at threshold'
}
@@ -16,12 +16,20 @@ export function prepareRetiredHistory(
try {
const domain = hasTable(db, 'journal_chat_state')
const heads = domain ? 'journal_chat_state' : 'journal_sessions'
const targetHead = db.prepare(`SELECT epoch FROM ${heads} WHERE session_id=?`).get(target)
if (
!db.prepare(`SELECT 1 FROM ${heads} WHERE session_id=? AND epoch IS NOT NULL`).get(target)
typeof targetHead?.epoch !== 'string' ||
targetHead.epoch === epoch ||
source === target ||
!Number.isSafeInteger(limit) ||
limit <= 0
) {
throw new Error('retired fixture target has no ready current head')
throw new Error(
'retired fixture requires an unrelated ready target, retired epoch and positive count'
)
}
db.exec('BEGIN IMMEDIATE')
let generation = 0
if (domain) {
const columns = db
.prepare('PRAGMA table_info(journal_items)')
@@ -41,20 +49,39 @@ export function prepareRetiredHistory(
limit
)
} else {
db.prepare(`INSERT INTO journal_rows(session_id,epoch,seq,ts,row_json)
SELECT ?,?,seq,ts,row_json FROM (
SELECT seq,ts,row_json,ROW_NUMBER() OVER (
PARTITION BY json_extract(row_json,'$.itemId') ORDER BY seq DESC) AS latest
FROM journal_rows WHERE session_id=? AND epoch=
(SELECT epoch FROM journal_sessions WHERE session_id=?) AND json_extract(row_json,'$.kind')='item'
) WHERE latest=1 ORDER BY seq LIMIT ?`).run(target, epoch, source, source, limit)
const head = db
.prepare(`SELECT head.* FROM journal_outcome_epochs head JOIN journal_sessions live
ON live.session_id=head.session_id AND live.epoch=head.epoch WHERE head.session_id=?`)
.get(source)
if (!head || head.prefix_seq !== 0 || Number(head.format) < 1 || Number(head.format) > 3) {
throw new Error('retired fixture requires a supported complete outcome source')
}
// No current epoch points at this generation: the main arm's existing sweeper owns its cleanup.
generation = Number(
db
.prepare('INSERT INTO journal_outcome_generations(session_id,epoch) VALUES (?,?)')
.run(target, epoch).lastInsertRowid
)
db.prepare(`INSERT INTO journal_outcomes(generation,seq,field,entry_key,format,ordinal,value_json,deleted)
SELECT ?,seq,field,entry_key,format,ordinal,value_json,deleted FROM (
SELECT *,ROW_NUMBER() OVER (PARTITION BY entry_key ORDER BY seq DESC) AS latest
FROM journal_outcomes WHERE generation=? AND seq<=? AND field='items'
) WHERE latest=1 AND deleted=0 ORDER BY ordinal,entry_key LIMIT ?`).run(
generation,
head.generation,
head.through_seq,
limit
)
}
const table = domain ? 'journal_items' : 'journal_rows'
const payload = domain ? 'payload_json' : 'row_json'
const result = db
.prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(${payload})),0) AS bytes
FROM ${table} WHERE session_id=? AND epoch=?`)
.get(target, epoch)!
.prepare(
domain
? `SELECT count(*) AS n,COALESCE(sum(octet_length(payload_json)),0) AS bytes
FROM journal_items WHERE session_id=? AND epoch=?`
: `SELECT count(*) AS n,COALESCE(sum(octet_length(value_json)),0) AS bytes
FROM journal_outcomes WHERE generation=?`
)
.get(...(domain ? [target, epoch] : [generation]))!
if (Number(result.n) !== limit) {
throw new Error(`retired fixture copied ${String(result.n)} rows, expected ${limit}`)
}
@@ -73,13 +100,18 @@ export function retiredHistory(file: string, sessionId: string): RetiredHistory
const db = openReadOnly(file)
try {
const domain = hasTable(db, 'journal_chat_state')
const table = domain ? 'journal_items' : 'journal_rows'
const heads = domain ? 'journal_chat_state' : 'journal_sessions'
const payload = domain ? 'payload_json' : 'row_json'
const row = db
.prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(item.${payload})),0) AS bytes
FROM ${table} item JOIN ${heads} head ON item.session_id=head.session_id
WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch`)
.prepare(
domain
? `SELECT count(*) AS n,COALESCE(sum(octet_length(item.payload_json)),0) AS bytes
FROM journal_items item JOIN journal_chat_state head ON item.session_id=head.session_id
WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch`
: `SELECT count(*) AS n,COALESCE(sum(octet_length(item.value_json)),0) AS bytes
FROM journal_outcomes item JOIN journal_outcome_generations retired ON retired.generation=item.generation
LEFT JOIN journal_outcome_epochs head ON head.session_id=retired.session_id
AND head.epoch=retired.epoch AND head.generation=retired.generation
WHERE retired.session_id=? AND head.session_id IS NULL`
)
.get(sessionId)!
return { rows: Number(row.n), payloadBytes: Number(row.bytes) }
} finally {
@@ -1,6 +1,81 @@
// Read-only source sizes and persisted outline keys, queried outside either product arm.
import { openReadOnly, hasTable } from './native-chat-4a-sqlite-observer'
function openItemSource(file: string, sessionId: string) {
const db = openReadOnly(file)
try {
const domain = hasTable(db, 'journal_chat_state')
if (!domain && !hasTable(db, 'journal_outcomes')) {
throw new Error('unsupported current-item storage; observation unexercised')
}
const head = domain
? `SELECT session_id,epoch,epoch AS source,
(SELECT COALESCE(max(ordinal),0) FROM journal_items
WHERE session_id=head.session_id AND epoch=head.epoch) AS position
FROM journal_chat_state head WHERE session_id=?`
: `SELECT head.session_id,head.epoch,head.generation AS source,head.through_seq AS position
FROM journal_outcome_epochs head JOIN journal_sessions live
ON live.session_id=head.session_id AND live.epoch=head.epoch
WHERE head.session_id=? AND head.prefix_seq=0 AND head.format BETWEEN 1 AND 3`
const baseline = db.prepare(head).get(sessionId)
if (
!baseline ||
typeof baseline.epoch !== 'string' ||
typeof baseline.position !== 'number' ||
(typeof baseline.source !== 'string' && typeof baseline.source !== 'number')
) {
throw new Error('no supported current item source; observation unexercised')
}
const tail = domain
? `SELECT head.epoch,head.source,item.ordinal AS position,item.item_id AS id,
'items' AS field,0 AS deleted,item.rowid AS stored_row
FROM journal_items item JOIN head ON item.session_id=head.session_id AND item.epoch=head.epoch
WHERE item.ordinal>? ORDER BY item.ordinal DESC LIMIT ?`
: `SELECT head.epoch,head.source,item.seq AS position,json_extract(item.entry_key,'$') AS id,
item.field,item.deleted,item.rowid AS stored_row
FROM journal_outcomes item JOIN head ON item.generation=head.source
WHERE item.seq>? AND item.seq<=head.position ORDER BY item.seq DESC LIMIT ?`
return {
db,
epoch: baseline.epoch,
source: baseline.source,
position: baseline.position,
// Sort only bounded metadata; fetch bodies after obsolete revisions/deletions are excluded.
items: `WITH head AS (${head}),recent AS (${tail}),latest AS (
SELECT *,ROW_NUMBER() OVER (PARTITION BY field,id ORDER BY position DESC) AS rank FROM recent)
SELECT latest.*,stored.${domain ? 'payload_json' : 'value_json'} AS json FROM latest
JOIN ${domain ? 'journal_items' : 'journal_outcomes'} stored ON stored.rowid=latest.stored_row
WHERE latest.rank=1 AND latest.field='items' AND typeof(latest.id)='text'
AND latest.deleted=0 AND latest.epoch=? AND latest.source=?`,
close: () => db.close()
}
} catch (error) {
db.close()
throw error
}
}
/** Current burst identities only; each read releases its snapshot before the caller dispatches. */
export function runningToolsObserver(file: string, sessionId: string) {
const source = openItemSource(file, sessionId)
try {
const query = source.db.prepare(`${source.items}
AND json_extract(json,'$.body.kind')='tool-call' AND json_extract(json,'$.body.state')='running'`)
return {
epoch: source.epoch,
position: source.position,
read: () =>
query
.all(sessionId, source.position, 40, source.epoch, source.source)
.map((item) => String(item.id)),
close: source.close
}
} catch (error) {
source.close()
throw error
}
}
export type OutlineCacheState = {
epoch: string
sequence: number
@@ -15,7 +90,7 @@ export type OutlineCacheState = {
export function outlineCacheState(file: string, sessionId: string): OutlineCacheState | null {
const db = openReadOnly(file)
try {
if (!hasTable(db, 'journal_outline_cache')) {
if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) {
return null
}
const row = db
@@ -49,6 +124,9 @@ export function outlineCacheState(file: string, sessionId: string): OutlineCache
export function outlineCacheContains(file: string, sessionId: string, value: unknown): boolean {
const db = openReadOnly(file)
try {
if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) {
return false
}
return Boolean(
db
.prepare(`SELECT 1 FROM journal_outline_cache cache JOIN journal_chat_state head
@@ -62,23 +140,23 @@ export function outlineCacheContains(file: string, sessionId: string, value: unk
}
}
/** Read only the latest eight stored rows; SQLite returns the current ASCII stream's byte count, never its growing body. */
/** At most 40 source records, newest revision per key; SQLite returns bytes, never the growing ASCII body. */
export function streamTextObserver(file: string, sessionId: string, marker: string) {
const db = openReadOnly(file)
const source = hasTable(db, 'journal_chat_state')
? `SELECT payload_json AS json FROM journal_items WHERE session_id=? AND epoch=
(SELECT epoch FROM journal_chat_state WHERE session_id=?) ORDER BY ordinal DESC LIMIT 8`
: `SELECT row_json AS json FROM journal_rows WHERE session_id=? AND epoch=
(SELECT epoch FROM journal_sessions WHERE session_id=?) ORDER BY seq DESC LIMIT 8`
const query =
db.prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes,
json_extract(json,'$.body.state') AS state FROM (${source})
WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? LIMIT 1`)
return {
read: () => {
const row = query.get(sessionId, sessionId, marker, marker)
return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' }
},
close: () => db.close()
const source = openItemSource(file, sessionId)
try {
const query = source.db
.prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes,
json_extract(json,'$.body.state') AS state FROM (${source.items})
WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? ORDER BY position DESC LIMIT 1`)
return {
read: () => {
const row = query.get(sessionId, -1, 40, source.epoch, source.source, marker, marker)
return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' }
},
close: source.close
}
} catch (error) {
source.close()
throw error
}
}