diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-stream-contention.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-stream-contention.ts index 655f3b05f33..20769e2ad70 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-stream-contention.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-stream-contention.ts @@ -16,8 +16,8 @@ import { waitForIdle, waitForWorking } from './native-chat-4a-session' -import { chatDatabasePath, hasTable, openReadOnly } from './native-chat-4a-sqlite-observer' -import { streamTextObserver } from './native-chat-4a-source-observer' +import { chatDatabasePath } from './native-chat-4a-sqlite-observer' +import { streamTextObserver, runningToolsObserver } from './native-chat-4a-source-observer' import { replyPrompt, streamPrompt, @@ -38,55 +38,6 @@ export async function competingSend( return { hostMs: answer.hostMs, durableMs: Date.now() - sentAt } } -// Only post-baseline rows, newest revision per ID: old running journal revisions cannot prove overlap. -function runningToolsObserver(file: string, sessionId: string) { - const db = openReadOnly(file) - try { - const domain = hasTable(db, 'journal_chat_state') - const position = domain ? 'ordinal' : 'seq' - const source = `${domain ? 'journal_items' : 'journal_rows'} WHERE session_id=? AND epoch= - (SELECT epoch FROM ${domain ? 'journal_chat_state' : 'journal_sessions'} WHERE session_id=?)` - const baseline = db - .prepare( - `SELECT epoch,${position} AS position FROM ${source} ORDER BY ${position} DESC LIMIT 1` - ) - .get(sessionId, sessionId) - if (!baseline || typeof baseline.epoch !== 'string' || typeof baseline.position !== 'number') { - throw new Error('no current source boundary; live contention not exercised') - } - const tail = `FROM ${source} AND ${position}>? ORDER BY ${position} DESC LIMIT 40` - const query = db.prepare( - domain - ? `SELECT epoch,item_id AS id,kind,lifecycle_state AS state ${tail}` - : `SELECT epoch,json_extract(row_json,'$.itemId') AS id, - json_extract(row_json,'$.body.kind') AS kind,json_extract(row_json,'$.body.state') AS state - FROM (SELECT epoch,seq,row_json ${tail}) ORDER BY seq DESC` - ) - return { - epoch: baseline.epoch, - position: baseline.position, - read: () => { - const seen = new Set() - const running: string[] = [] - for (const item of query.all(sessionId, sessionId, baseline.position)) { - if (item.epoch !== baseline.epoch || typeof item.id !== 'string' || seen.has(item.id)) { - continue - } - seen.add(item.id) - if (item.kind === 'tool-call' && item.state === 'running') { - running.push(item.id) - } - } - return running - }, - close: () => db.close() - } - } catch (error) { - db.close() - throw error - } -} - export async function phaseSample( ctx: RunContext, shape: StreamCase['shape'], diff --git a/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts index 86d1d970f52..87e2226f7d5 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-ops-stream.ts @@ -213,7 +213,7 @@ export async function streamingRows(ctx: RunContext): Promise { streamBytes: bytes + 1024 ** 2, largeItems, observation: - 'outside read-only SQLite current text, latest eight rows, every 100 ms; adds reader load', + 'outside read-only SQLite current text, latest 40 records per source, every 100 ms; adds reader load', concurrent: 'send to another chat, large-chat tail read and Stop start together at threshold' } diff --git a/config/scripts/performance/native-chat/native-chat-4a-retired-fixture.ts b/config/scripts/performance/native-chat/native-chat-4a-retired-fixture.ts index 7f1dad13d59..f311fd0c016 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-retired-fixture.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-retired-fixture.ts @@ -16,12 +16,20 @@ export function prepareRetiredHistory( try { const domain = hasTable(db, 'journal_chat_state') const heads = domain ? 'journal_chat_state' : 'journal_sessions' + const targetHead = db.prepare(`SELECT epoch FROM ${heads} WHERE session_id=?`).get(target) if ( - !db.prepare(`SELECT 1 FROM ${heads} WHERE session_id=? AND epoch IS NOT NULL`).get(target) + typeof targetHead?.epoch !== 'string' || + targetHead.epoch === epoch || + source === target || + !Number.isSafeInteger(limit) || + limit <= 0 ) { - throw new Error('retired fixture target has no ready current head') + throw new Error( + 'retired fixture requires an unrelated ready target, retired epoch and positive count' + ) } db.exec('BEGIN IMMEDIATE') + let generation = 0 if (domain) { const columns = db .prepare('PRAGMA table_info(journal_items)') @@ -41,20 +49,39 @@ export function prepareRetiredHistory( limit ) } else { - db.prepare(`INSERT INTO journal_rows(session_id,epoch,seq,ts,row_json) - SELECT ?,?,seq,ts,row_json FROM ( - SELECT seq,ts,row_json,ROW_NUMBER() OVER ( - PARTITION BY json_extract(row_json,'$.itemId') ORDER BY seq DESC) AS latest - FROM journal_rows WHERE session_id=? AND epoch= - (SELECT epoch FROM journal_sessions WHERE session_id=?) AND json_extract(row_json,'$.kind')='item' - ) WHERE latest=1 ORDER BY seq LIMIT ?`).run(target, epoch, source, source, limit) + const head = db + .prepare(`SELECT head.* FROM journal_outcome_epochs head JOIN journal_sessions live + ON live.session_id=head.session_id AND live.epoch=head.epoch WHERE head.session_id=?`) + .get(source) + if (!head || head.prefix_seq !== 0 || Number(head.format) < 1 || Number(head.format) > 3) { + throw new Error('retired fixture requires a supported complete outcome source') + } + // No current epoch points at this generation: the main arm's existing sweeper owns its cleanup. + generation = Number( + db + .prepare('INSERT INTO journal_outcome_generations(session_id,epoch) VALUES (?,?)') + .run(target, epoch).lastInsertRowid + ) + db.prepare(`INSERT INTO journal_outcomes(generation,seq,field,entry_key,format,ordinal,value_json,deleted) + SELECT ?,seq,field,entry_key,format,ordinal,value_json,deleted FROM ( + SELECT *,ROW_NUMBER() OVER (PARTITION BY entry_key ORDER BY seq DESC) AS latest + FROM journal_outcomes WHERE generation=? AND seq<=? AND field='items' + ) WHERE latest=1 AND deleted=0 ORDER BY ordinal,entry_key LIMIT ?`).run( + generation, + head.generation, + head.through_seq, + limit + ) } - const table = domain ? 'journal_items' : 'journal_rows' - const payload = domain ? 'payload_json' : 'row_json' const result = db - .prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(${payload})),0) AS bytes - FROM ${table} WHERE session_id=? AND epoch=?`) - .get(target, epoch)! + .prepare( + domain + ? `SELECT count(*) AS n,COALESCE(sum(octet_length(payload_json)),0) AS bytes + FROM journal_items WHERE session_id=? AND epoch=?` + : `SELECT count(*) AS n,COALESCE(sum(octet_length(value_json)),0) AS bytes + FROM journal_outcomes WHERE generation=?` + ) + .get(...(domain ? [target, epoch] : [generation]))! if (Number(result.n) !== limit) { throw new Error(`retired fixture copied ${String(result.n)} rows, expected ${limit}`) } @@ -73,13 +100,18 @@ export function retiredHistory(file: string, sessionId: string): RetiredHistory const db = openReadOnly(file) try { const domain = hasTable(db, 'journal_chat_state') - const table = domain ? 'journal_items' : 'journal_rows' - const heads = domain ? 'journal_chat_state' : 'journal_sessions' - const payload = domain ? 'payload_json' : 'row_json' const row = db - .prepare(`SELECT count(*) AS n,COALESCE(sum(octet_length(item.${payload})),0) AS bytes - FROM ${table} item JOIN ${heads} head ON item.session_id=head.session_id - WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch`) + .prepare( + domain + ? `SELECT count(*) AS n,COALESCE(sum(octet_length(item.payload_json)),0) AS bytes + FROM journal_items item JOIN journal_chat_state head ON item.session_id=head.session_id + WHERE item.session_id=? AND head.epoch IS NOT NULL AND item.epoch<>head.epoch` + : `SELECT count(*) AS n,COALESCE(sum(octet_length(item.value_json)),0) AS bytes + FROM journal_outcomes item JOIN journal_outcome_generations retired ON retired.generation=item.generation + LEFT JOIN journal_outcome_epochs head ON head.session_id=retired.session_id + AND head.epoch=retired.epoch AND head.generation=retired.generation + WHERE retired.session_id=? AND head.session_id IS NULL` + ) .get(sessionId)! return { rows: Number(row.n), payloadBytes: Number(row.bytes) } } finally { diff --git a/config/scripts/performance/native-chat/native-chat-4a-source-observer.ts b/config/scripts/performance/native-chat/native-chat-4a-source-observer.ts index 379084f9260..133c7139365 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-source-observer.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-source-observer.ts @@ -1,6 +1,81 @@ // Read-only source sizes and persisted outline keys, queried outside either product arm. import { openReadOnly, hasTable } from './native-chat-4a-sqlite-observer' +function openItemSource(file: string, sessionId: string) { + const db = openReadOnly(file) + try { + const domain = hasTable(db, 'journal_chat_state') + if (!domain && !hasTable(db, 'journal_outcomes')) { + throw new Error('unsupported current-item storage; observation unexercised') + } + const head = domain + ? `SELECT session_id,epoch,epoch AS source, + (SELECT COALESCE(max(ordinal),0) FROM journal_items + WHERE session_id=head.session_id AND epoch=head.epoch) AS position + FROM journal_chat_state head WHERE session_id=?` + : `SELECT head.session_id,head.epoch,head.generation AS source,head.through_seq AS position + FROM journal_outcome_epochs head JOIN journal_sessions live + ON live.session_id=head.session_id AND live.epoch=head.epoch + WHERE head.session_id=? AND head.prefix_seq=0 AND head.format BETWEEN 1 AND 3` + const baseline = db.prepare(head).get(sessionId) + if ( + !baseline || + typeof baseline.epoch !== 'string' || + typeof baseline.position !== 'number' || + (typeof baseline.source !== 'string' && typeof baseline.source !== 'number') + ) { + throw new Error('no supported current item source; observation unexercised') + } + const tail = domain + ? `SELECT head.epoch,head.source,item.ordinal AS position,item.item_id AS id, + 'items' AS field,0 AS deleted,item.rowid AS stored_row + FROM journal_items item JOIN head ON item.session_id=head.session_id AND item.epoch=head.epoch + WHERE item.ordinal>? ORDER BY item.ordinal DESC LIMIT ?` + : `SELECT head.epoch,head.source,item.seq AS position,json_extract(item.entry_key,'$') AS id, + item.field,item.deleted,item.rowid AS stored_row + FROM journal_outcomes item JOIN head ON item.generation=head.source + WHERE item.seq>? AND item.seq<=head.position ORDER BY item.seq DESC LIMIT ?` + return { + db, + epoch: baseline.epoch, + source: baseline.source, + position: baseline.position, + // Sort only bounded metadata; fetch bodies after obsolete revisions/deletions are excluded. + items: `WITH head AS (${head}),recent AS (${tail}),latest AS ( + SELECT *,ROW_NUMBER() OVER (PARTITION BY field,id ORDER BY position DESC) AS rank FROM recent) + SELECT latest.*,stored.${domain ? 'payload_json' : 'value_json'} AS json FROM latest + JOIN ${domain ? 'journal_items' : 'journal_outcomes'} stored ON stored.rowid=latest.stored_row + WHERE latest.rank=1 AND latest.field='items' AND typeof(latest.id)='text' + AND latest.deleted=0 AND latest.epoch=? AND latest.source=?`, + close: () => db.close() + } + } catch (error) { + db.close() + throw error + } +} + +/** Current burst identities only; each read releases its snapshot before the caller dispatches. */ +export function runningToolsObserver(file: string, sessionId: string) { + const source = openItemSource(file, sessionId) + try { + const query = source.db.prepare(`${source.items} + AND json_extract(json,'$.body.kind')='tool-call' AND json_extract(json,'$.body.state')='running'`) + return { + epoch: source.epoch, + position: source.position, + read: () => + query + .all(sessionId, source.position, 40, source.epoch, source.source) + .map((item) => String(item.id)), + close: source.close + } + } catch (error) { + source.close() + throw error + } +} + export type OutlineCacheState = { epoch: string sequence: number @@ -15,7 +90,7 @@ export type OutlineCacheState = { export function outlineCacheState(file: string, sessionId: string): OutlineCacheState | null { const db = openReadOnly(file) try { - if (!hasTable(db, 'journal_outline_cache')) { + if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) { return null } const row = db @@ -49,6 +124,9 @@ export function outlineCacheState(file: string, sessionId: string): OutlineCache export function outlineCacheContains(file: string, sessionId: string, value: unknown): boolean { const db = openReadOnly(file) try { + if (!hasTable(db, 'journal_chat_state') || !hasTable(db, 'journal_outline_cache')) { + return false + } return Boolean( db .prepare(`SELECT 1 FROM journal_outline_cache cache JOIN journal_chat_state head @@ -62,23 +140,23 @@ export function outlineCacheContains(file: string, sessionId: string, value: unk } } -/** Read only the latest eight stored rows; SQLite returns the current ASCII stream's byte count, never its growing body. */ +/** At most 40 source records, newest revision per key; SQLite returns bytes, never the growing ASCII body. */ export function streamTextObserver(file: string, sessionId: string, marker: string) { - const db = openReadOnly(file) - const source = hasTable(db, 'journal_chat_state') - ? `SELECT payload_json AS json FROM journal_items WHERE session_id=? AND epoch= - (SELECT epoch FROM journal_chat_state WHERE session_id=?) ORDER BY ordinal DESC LIMIT 8` - : `SELECT row_json AS json FROM journal_rows WHERE session_id=? AND epoch= - (SELECT epoch FROM journal_sessions WHERE session_id=?) ORDER BY seq DESC LIMIT 8` - const query = - db.prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes, - json_extract(json,'$.body.state') AS state FROM (${source}) - WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? LIMIT 1`) - return { - read: () => { - const row = query.get(sessionId, sessionId, marker, marker) - return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' } - }, - close: () => db.close() + const source = openItemSource(file, sessionId) + try { + const query = source.db + .prepare(`SELECT octet_length(json_extract(json,'$.body.blocks[0].text')) AS bytes, + json_extract(json,'$.body.state') AS state FROM (${source.items}) + WHERE substr(json_extract(json,'$.body.blocks[0].text'),1,length(?))=? ORDER BY position DESC LIMIT 1`) + return { + read: () => { + const row = query.get(sessionId, -1, 40, source.epoch, source.source, marker, marker) + return { bytes: Number(row?.bytes ?? 0), completed: row?.state === 'completed' } + }, + close: source.close + } + } catch (error) { + source.close() + throw error } }