mirror of
https://github.com/stablyai/orca.git
synced 2026-10-29 00:03:37 +00:00
fix(agent-status): filter harness-injected turns by tag shape, not a literal list
Harness machinery turns (task-notification, agent-message, bash-input/stdout, system-reminder, …) were leaking into prompt-derived UI as if the user typed them — hook-status labels, native-chat bubbles, and AI Vault session titles. The literal tag blocklist went stale twice in three months because the harness adds tags faster than we chase them. Generalize the text classifier to match the harness's tag *shape*: any prompt beginning with a lowercase multi-word kebab tag (<task-notification>, <agent-message …>, <bash-input>, …) is machinery. Underscore tags (<user_query>) are deliberately excluded — Grok wraps real typed prompts in them. Explicit prefixes remain only for the single-word tag (<channel source=) and the prose wrappers the harness uses. Add a structural drop at the native-chat decoder (user turns marked isMeta/isSynthetic/isCompactSummary, unless they carry tool results) and gate AI Vault title seeding on the shared classifier, since task-notifications carry no isMeta.
This commit is contained in:
committed by
Brennan Benson
parent
2b7c36474d
commit
60ff5fece5
@@ -0,0 +1,56 @@
|
||||
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { scanAiVaultSessions } from './session-scanner'
|
||||
import { isolatedScanRoots, jsonLines } from './session-scanner-test-fixtures'
|
||||
|
||||
let tempRoots: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true })))
|
||||
tempRoots = []
|
||||
})
|
||||
|
||||
describe('scanAiVaultSessions harness-injected title seeding', () => {
|
||||
it('keeps harness-injected Claude turns without isMeta out of session titles', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-claude-injected-title-'))
|
||||
tempRoots.push(root)
|
||||
const roots = isolatedScanRoots(root)
|
||||
await mkdir(join(roots.claudeProjectsDir, 'project'), { recursive: true })
|
||||
|
||||
await writeFile(
|
||||
join(roots.claudeProjectsDir, 'project', 'injected-first.jsonl'),
|
||||
jsonLines([
|
||||
{
|
||||
type: 'user',
|
||||
sessionId: 'injected-first',
|
||||
timestamp: '2026-06-11T10:00:00.000Z',
|
||||
cwd: '/repo/app',
|
||||
// Task notifications carry no isMeta — only their text marks them.
|
||||
message: {
|
||||
role: 'user',
|
||||
content: '<task-notification> <task-id>abc</task-id> <status>completed</status>'
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
sessionId: 'injected-first',
|
||||
timestamp: '2026-06-11T10:00:01.000Z',
|
||||
cwd: '/repo/app',
|
||||
message: { role: 'user', content: 'Fix the sidebar label bug' }
|
||||
}
|
||||
])
|
||||
)
|
||||
|
||||
const result = await scanAiVaultSessions({
|
||||
...roots,
|
||||
platform: 'darwin'
|
||||
})
|
||||
|
||||
expect(result.issues).toEqual([])
|
||||
expect(result.sessions).toHaveLength(1)
|
||||
expect(result.sessions[0]?.title).toBe('Fix the sidebar label bug')
|
||||
})
|
||||
})
|
||||
@@ -2,6 +2,7 @@ import { createReadStream } from 'node:fs'
|
||||
import { createInterface } from 'node:readline'
|
||||
import type { AiVaultSession } from '../../shared/ai-vault-types'
|
||||
import { LOCAL_EXECUTION_HOST_ID, type ExecutionHostId } from '../../shared/execution-host'
|
||||
import { isHarnessInjectedUserTurnText } from '../../shared/harness-injected-user-turns'
|
||||
import type {
|
||||
FileWithMtime,
|
||||
ResumableSessionParseState,
|
||||
@@ -119,8 +120,10 @@ export function consumeClaudeSessionLine(state: ClaudeSessionParseState, line: s
|
||||
const title = extractMessageText(record.message)
|
||||
addPreviewContent(accumulator, 'user', asRecord(record.message)?.content, record.timestamp)
|
||||
if (title) {
|
||||
// Meta prompts (injected context) only seed the last-resort title.
|
||||
if (record.isMeta === true) {
|
||||
// Meta prompts (injected context) only seed the last-resort title. Some
|
||||
// injected turns (task notifications) carry no isMeta, so also gate on
|
||||
// the shared machinery-text classifier.
|
||||
if (record.isMeta === true || isHarnessInjectedUserTurnText(title)) {
|
||||
state.metaTitle ??= title
|
||||
} else {
|
||||
state.firstUserTitle ??= title
|
||||
|
||||
@@ -26,6 +26,15 @@ export function decodeClaudeTranscriptLine(
|
||||
if (blocks.length === 0) {
|
||||
return null
|
||||
}
|
||||
// Why: Claude structurally marks injected turns, but tool-result records are
|
||||
// genuine output and must remain visible even when the containing turn is meta.
|
||||
if (
|
||||
role === 'user' &&
|
||||
(record.isMeta === true || record.isSynthetic === true || record.isCompactSummary === true) &&
|
||||
!blocks.some((block) => block.type === 'tool-result')
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const messageId = extractString(record.uuid) ?? extractString(message?.id)
|
||||
return {
|
||||
id: messageId ?? fallbackId,
|
||||
|
||||
@@ -87,6 +87,53 @@ describe('readNativeChatTranscript (claude)', () => {
|
||||
expect(toolResult?.blocks[0]).toEqual({ type: 'tool-result', output: 'file-a\nfile-b' })
|
||||
})
|
||||
|
||||
it('drops structurally marked injected user turns but keeps their tool results', async () => {
|
||||
const filePath = await writeFixture('orca-native-chat-claude-meta-', [
|
||||
{
|
||||
type: 'user',
|
||||
uuid: 'u-real',
|
||||
timestamp: '2026-06-01T10:00:00.000Z',
|
||||
message: { role: 'user', content: 'fix the login bug' }
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
uuid: 'u-meta',
|
||||
isMeta: true,
|
||||
timestamp: '2026-06-01T10:00:01.000Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: 'Another Claude session sent a message:\n<agent-message from="reviewer">hi'
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
uuid: 'u-compact',
|
||||
isCompactSummary: true,
|
||||
timestamp: '2026-06-01T10:00:02.000Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: 'This session is being continued from a previous conversation.'
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'user',
|
||||
uuid: 'u-meta-toolresult',
|
||||
isMeta: true,
|
||||
timestamp: '2026-06-01T10:00:03.000Z',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [{ type: 'tool_result', content: 'ok', is_error: false }]
|
||||
}
|
||||
}
|
||||
])
|
||||
const result = await readNativeChatTranscript('claude', 'sess', { filePath })
|
||||
if (!('messages' in result)) {
|
||||
throw new Error('expected messages')
|
||||
}
|
||||
expect(result.messages.map((m) => m.id)).toEqual(['u-real', 'u-meta-toolresult'])
|
||||
expect(result.messages[1].role).toBe('tool')
|
||||
})
|
||||
|
||||
it('marks thinking-only assistant content as a reasoning surface', async () => {
|
||||
const filePath = await writeFixture('orca-native-chat-claude-think-', [
|
||||
{
|
||||
|
||||
@@ -3,10 +3,10 @@ import { isHarnessInjectedUserTurnText } from '../../../../shared/harness-inject
|
||||
|
||||
// Why: harness machinery turns land in the transcript but are not real user
|
||||
// messages, so the chat filters them out (they were confusingly rendered as
|
||||
// the user's own bubbles). The prefix list lives in
|
||||
// the user's own bubbles). The classifier lives in
|
||||
// src/shared/harness-injected-user-turns.ts, shared with the agent-status
|
||||
// prompt pipeline. Mirrors the mobile predicate in
|
||||
// mobile/src/session/mobile-native-chat-noise.ts.
|
||||
// prompt pipeline; structurally marked turns (isMeta etc.) are already
|
||||
// dropped at decode in src/main/native-chat/transcript-line-decoders.ts.
|
||||
|
||||
function messageText(message: NativeChatMessage): string {
|
||||
return message.blocks
|
||||
|
||||
@@ -3,21 +3,59 @@ import { describe, expect, it } from 'vitest'
|
||||
import { isHarnessInjectedUserTurnText } from './harness-injected-user-turns'
|
||||
|
||||
describe('isHarnessInjectedUserTurnText', () => {
|
||||
it('matches harness machinery turns by prefix', () => {
|
||||
it('matches every known harness tag, including attribute-carrying forms', () => {
|
||||
// Inventory mirrors the tags the harness itself classifies as machinery.
|
||||
const injected = [
|
||||
'<task-notification> <task-id>bzthj2b8r</task-id> <tool-use-id>toolu_01abc</tool-use-id>',
|
||||
'<task-notification summary="Agent finished">',
|
||||
'<system-reminder>context</system-reminder>',
|
||||
'<agent-message from="reviewer-A-r3b"> REVIEW the diff',
|
||||
'<teammate-message teammate_id="worker-1">status?',
|
||||
'<cross-session-message from="other session">hello',
|
||||
'<channel source="general">new post',
|
||||
'<fork-boilerplate>Your directive: ship it',
|
||||
'<user-memory-input>remember this',
|
||||
'<mcp-resource-update uri="db://x">',
|
||||
'<mcp-polling-update>tick',
|
||||
'<command-name>/review</command-name>',
|
||||
'<command-message>review</command-message>',
|
||||
'<command-args>--fix</command-args>',
|
||||
'<local-command-stdout>ok</local-command-stdout>',
|
||||
'<local-command-stderr>boom</local-command-stderr>',
|
||||
'<local-command-caveat>Caveat: tool output</local-command-caveat>',
|
||||
'<bash-input>ls</bash-input>',
|
||||
'<bash-stdout>file.txt</bash-stdout>',
|
||||
'<bash-stderr>err</bash-stderr>',
|
||||
'<user-prompt-submit-hook>hook context</user-prompt-submit-hook>'
|
||||
]
|
||||
for (const text of injected) {
|
||||
expect(isHarnessInjectedUserTurnText(text), text).toBe(true)
|
||||
}
|
||||
})
|
||||
|
||||
it('matches future kebab tags without a list update', () => {
|
||||
expect(isHarnessInjectedUserTurnText('<brand-new-harness-tag id="1">payload')).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('<queued-notice>\nlater')).toBe(true)
|
||||
})
|
||||
|
||||
it('matches harness prose wrappers and notices', () => {
|
||||
expect(
|
||||
isHarnessInjectedUserTurnText(
|
||||
'<task-notification> <task-id>bzthj2b8r</task-id> <tool-use-id>toolu_01abc</tool-use-id>'
|
||||
)
|
||||
isHarnessInjectedUserTurnText('A message arrived from teammate-b:\n<agent-message>hi')
|
||||
).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('<system-reminder>context</system-reminder>')).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('<command-name>/review</command-name>')).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('<local-command-stdout>ok</local-command-stdout>')).toBe(
|
||||
true
|
||||
)
|
||||
expect(
|
||||
isHarnessInjectedUserTurnText('Another Claude session sent a message:\n<agent-message>hi')
|
||||
).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('No response requested.')).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('[Request interrupted by user]')).toBe(true)
|
||||
expect(isHarnessInjectedUserTurnText('[Request interrupted by user for tool use]')).toBe(true)
|
||||
expect(
|
||||
isHarnessInjectedUserTurnText('This session is being continued from a previous conversation.')
|
||||
).toBe(true)
|
||||
expect(
|
||||
isHarnessInjectedUserTurnText(
|
||||
'Caveat: the messages below were generated by the user while running local commands.'
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('is case-insensitive and ignores surrounding whitespace', () => {
|
||||
@@ -33,4 +71,15 @@ describe('isHarnessInjectedUserTurnText', () => {
|
||||
expect(isHarnessInjectedUserTurnText('')).toBe(false)
|
||||
expect(isHarnessInjectedUserTurnText(' ')).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps single-word tag pastes and underscore wrappers', () => {
|
||||
// Grok wraps REAL typed prompts in <user_query> — never classify as noise.
|
||||
expect(isHarnessInjectedUserTurnText('<user_query>fix the bug</user_query>')).toBe(false)
|
||||
expect(isHarnessInjectedUserTurnText('<div class="x">pasted html</div>')).toBe(false)
|
||||
expect(isHarnessInjectedUserTurnText('<script>alert(1)</script> — why is this flagged?')).toBe(
|
||||
false
|
||||
)
|
||||
expect(isHarnessInjectedUserTurnText('<https://example.com/a-b> what is this?')).toBe(false)
|
||||
expect(isHarnessInjectedUserTurnText('<foo-bar@example.com> sent me this')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,29 +1,40 @@
|
||||
// Why: agent harnesses (Claude Code and its forks) inject machinery into the
|
||||
// conversation as user-role turns — background task notifications, system
|
||||
// reminders, slash-command envelopes, local-command output, interruption and
|
||||
// compaction notices. These fire user-prompt hooks and land in transcripts,
|
||||
// but they are not something the user typed, so prompt-derived UI must not
|
||||
// surface them.
|
||||
// reminders, inter-agent messages, slash-command envelopes, local-command
|
||||
// output, interruption and compaction notices. These fire user-prompt hooks
|
||||
// and land in transcripts, but they are not something the user typed, so
|
||||
// prompt-derived UI must not surface them.
|
||||
//
|
||||
// Every tag the harness injects is a lowercase multi-word kebab tag
|
||||
// (<task-notification>, <agent-message from=…>, <cross-session-message>, …),
|
||||
// so we match that shape instead of chasing individual tag names — the tag
|
||||
// set grows faster than a literal list stays current. Underscore tags
|
||||
// (<user_query>) are deliberately NOT matched: Grok wraps real typed prompts
|
||||
// in them. A prompt that begins with pasted kebab-tag markup (e.g. a
|
||||
// web-component snippet) is misclassified, costing only a stale status label.
|
||||
const HARNESS_KEBAB_TAG_PREFIX = /^<[a-z][a-z0-9]*(?:-[a-z0-9]+)+(?:[\s>]|$)/
|
||||
|
||||
// Injected shapes the kebab-tag rule cannot see: single-word attribute tags
|
||||
// and the prose sentences the harness wraps deliveries and notices in.
|
||||
const HARNESS_INJECTED_TURN_PREFIXES = [
|
||||
'<task-notification>',
|
||||
'<system-reminder>',
|
||||
'<local-command-stdout>',
|
||||
'<local-command-caveat>',
|
||||
'<command-name>',
|
||||
'<command-message>',
|
||||
'<command-args>',
|
||||
'<bash-',
|
||||
'<channel source=',
|
||||
'[request interrupted',
|
||||
'a message arrived from ',
|
||||
'another claude session sent a message',
|
||||
'no response requested.',
|
||||
'caveat: the messages below were generated by the user while running local commands',
|
||||
'this session is being continued from a previous conversation'
|
||||
]
|
||||
|
||||
/** True when prompt-like text is a harness-injected machinery turn rather
|
||||
* than something the user typed. Prefix match on trimmed, lowercased text. */
|
||||
* than something the user typed. Match on trimmed, lowercased text. */
|
||||
export function isHarnessInjectedUserTurnText(text: string): boolean {
|
||||
const normalized = text.trim().toLowerCase()
|
||||
if (normalized.length === 0) {
|
||||
return false
|
||||
}
|
||||
if (HARNESS_KEBAB_TAG_PREFIX.test(normalized)) {
|
||||
return true
|
||||
}
|
||||
return HARNESS_INJECTED_TURN_PREFIXES.some((prefix) => normalized.startsWith(prefix))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user