test: verify hook delivery and exit cleanup through observable state (#32994)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-10-23 18:04:50 -07:00
committed by GitHub
co-authored by OrcaWin
parent e38cf787e8
commit 69db315f52
5 changed files with 154 additions and 188 deletions
@@ -1,12 +1,9 @@
import { createServer } from 'node:http'
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { runProcess } from '@orca/process-host'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({ app: { getPath: () => '/tmp/userData' } }))
import { runVendorHook } from './vendor-hook-delivery.test-fixture'
import { getManagedScript as getClaudeManagedScript } from '../claude/hook-service'
import { getManagedScript as getCursorManagedScript } from '../cursor/hook-script'
@@ -17,74 +14,32 @@ const CLAUDE_SCRIPT_OPTIONS = {
}
const CURSOR_ENV = { CURSOR_VERSION: '2026.10.16', CURSOR_PROJECT_DIR: '/work/project' }
async function runHook(script: string, hookEnv: NodeJS.ProcessEnv) {
const requests: string[] = []
const server = createServer((request, response) => {
requests.push(request.url ?? '')
request.resume()
request.on('end', () => response.end('{}'))
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const dir = mkdtempSync(join(tmpdir(), 'orca-cursor-replay-'))
try {
const address = server.address()
if (!address || typeof address === 'string') {
throw new Error('Expected a loopback port')
}
const windows = process.platform === 'win32'
const scriptPath = join(dir, windows ? 'hook.cmd' : 'hook.sh')
writeFileSync(scriptPath, script)
const result = await runProcess({
program: windows ? scriptPath : '/bin/sh',
args: windows ? [] : [scriptPath],
input: '{"hook_event_name":"Stop"}',
env: {
...process.env,
CURSOR_VERSION: '',
CURSOR_PROJECT_DIR: '',
GROK_HOOK_EVENT: '',
DEVIN_SESSION_ID: '',
CLAUDE_JOB_DIR: '',
ORCA_AGENT_HOOK_ENDPOINT: '',
ORCA_AGENT_HOOK_PORT: String(address.port),
ORCA_AGENT_HOOK_TOKEN: 'test-token',
ORCA_PANE_KEY: 'tab:leaf',
...hookEnv
}
})
expect(result.code).toBe(0)
expect(result.timedOut).toBe(false)
expect(result.stdout.replaceAll('\r\n', '\n')).toBe('{}\n')
return requests
} finally {
await new Promise<void>((resolve, reject) =>
server.close((error) => (error ? reject(error) : resolve()))
)
rmSync(dir, { recursive: true, force: true })
}
}
describe('Cursor vendor hook replay guard', () => {
it('drops imported Claude events while preserving the hook protocol response', async () => {
expect(
await runHook(getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS), CURSOR_ENV)
(await runVendorHook(getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS), CURSOR_ENV))
.requests
).toEqual([])
})
it.each([{}, { CURSOR_VERSION: CURSOR_ENV.CURSOR_VERSION }, { CURSOR_PROJECT_DIR: '/p' }])(
'still delivers Claude hooks when only part of the Cursor environment is present: %j',
async (env) => {
expect(await runHook(getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS), env)).toEqual([
'/hook/claude'
])
expect(
(await runVendorHook(getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS), env)).requests
).toEqual(['/hook/claude'])
}
)
it('keeps Cursor native hooks reporting in the same environment', async () => {
expect(await runHook(getCursorManagedScript('local'), CURSOR_ENV)).toEqual(['/hook/cursor'])
expect((await runVendorHook(getCursorManagedScript('local'), CURSOR_ENV)).requests).toEqual([
'/hook/cursor'
])
})
it('does not suppress a hook service that did not opt in to the Claude import guard', async () => {
expect(await runHook(getClaudeManagedScript('local'), CURSOR_ENV)).toEqual(['/hook/claude'])
expect((await runVendorHook(getClaudeManagedScript('local'), CURSOR_ENV)).requests).toEqual([
'/hook/claude'
])
})
})
+34 -120
View File
@@ -1,142 +1,56 @@
import { spawnSync } from 'node:child_process'
import { chmodSync, existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: {
getPath: () => '/tmp/userData'
}
}))
vi.mock('electron', () => ({ app: { getPath: () => '/tmp/userData' } }))
import { getManagedScript as getClaudeManagedScript } from '../claude/hook-service'
import { getManagedScript as getCursorManagedScript } from '../cursor/hook-script'
import { runVendorHook } from './vendor-hook-delivery.test-fixture'
const POSIX_GROK_GUARD = 'if [ -n "$GROK_HOOK_EVENT" ]; then'
const WINDOWS_GROK_GUARD = 'if not "%GROK_HOOK_EVENT%"=="" goto :orca_agent_hook_drain_stdin'
const CLAUDE_SCRIPT_OPTIONS = {
skipWhenDevinImportsClaude: true,
skipWhenGrokImportsClaude: true
}
function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!
Object.defineProperty(process, 'platform', { value: platform, configurable: true })
try {
return run()
} finally {
Object.defineProperty(process, 'platform', descriptor)
}
}
function expectGuardBeforeTransport(
script: string,
guard: string,
response: string,
spool?: string
): void {
const guardIndex = script.indexOf(guard)
const responseIndex = script.indexOf(response)
expect(responseIndex).toBeGreaterThan(-1)
expect(guardIndex).toBeGreaterThan(responseIndex)
expect(guardIndex).toBeLessThan(script.indexOf('curl'))
if (spool) {
expect(guardIndex).toBeLessThan(script.indexOf(spool))
}
}
function runPosixHook(
script: string,
grokHookEvent: string
): {
curlCalled: boolean
stdout: string
} {
const dir = mkdtempSync(join(tmpdir(), 'orca-grok-replay-'))
const scriptPath = join(dir, 'hook.sh')
const curlPath = join(dir, 'curl')
const curlLog = join(dir, 'curl.log')
try {
writeFileSync(scriptPath, script)
writeFileSync(
curlPath,
'#!/bin/sh\n{ command -p cat 2>/dev/null || cat; } >/dev/null\nprintf "called\\n" >> "$CURL_LOG"\n'
)
chmodSync(scriptPath, 0o755)
chmodSync(curlPath, 0o755)
const result = spawnSync('/bin/sh', [scriptPath], {
encoding: 'utf8',
input: '{"hook_event_name":"Stop"}',
timeout: 5000,
env: {
...process.env,
PATH: `${dir}:${process.env.PATH ?? ''}`,
CURL_LOG: curlLog,
GROK_HOOK_EVENT: grokHookEvent,
DEVIN_PROJECT_DIR: '',
CLAUDE_JOB_DIR: '',
ORCA_CURSOR_HOOK_RESPONSE: '',
ORCA_AGENT_HOOK_ENDPOINT: '',
ORCA_AGENT_HOOK_PORT: '1234',
ORCA_AGENT_HOOK_TOKEN: 'token',
ORCA_PANE_KEY: 'tab:leaf'
}
})
expect(result.error).toBeUndefined()
expect(result.status).toBe(0)
return { curlCalled: existsSync(curlLog), stdout: result.stdout }
} finally {
rmSync(dir, { recursive: true, force: true })
}
}
describe('Grok vendor hook replay guard', () => {
it('precedes spooling and HTTP in the generated POSIX Claude and Cursor scripts', () => {
const claude = getClaudeManagedScript('posix', CLAUDE_SCRIPT_OPTIONS)
const cursor = getCursorManagedScript('posix')
expectGuardBeforeTransport(claude, POSIX_GROK_GUARD, 'printf "{}\\n"', 'spool_hook_event')
expectGuardBeforeTransport(cursor, POSIX_GROK_GUARD, 'printf "{}\\n"', 'spool_hook_event')
it('drops Grok-replayed hooks without suppressing their protocol response', async () => {
for (const script of [
getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS),
getCursorManagedScript('local')
]) {
expect((await runVendorHook(script, { GROK_HOOK_EVENT: 'Stop' })).requests).toEqual([])
}
})
it('precedes HTTP while preserving fail-open output in generated Windows scripts', () => {
const { claude, cursor } = withPlatform('win32', () => ({
claude: getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS),
cursor: getCursorManagedScript('local')
}))
expectGuardBeforeTransport(claude, WINDOWS_GROK_GUARD, 'echo {}')
expectGuardBeforeTransport(cursor, WINDOWS_GROK_GUARD, '(echo {})')
const backgroundWorkerGuardIndex = claude.indexOf('CLAUDE_JOB_DIR')
expect(backgroundWorkerGuardIndex).toBeGreaterThan(-1)
expect(backgroundWorkerGuardIndex).toBeLessThan(claude.indexOf(WINDOWS_GROK_GUARD))
it('leaves non-Grok hook delivery unchanged', async () => {
for (const [source, script] of [
['claude', getClaudeManagedScript('local', CLAUDE_SCRIPT_OPTIONS)],
['cursor', getCursorManagedScript('local')]
]) {
expect((await runVendorHook(script, { GROK_HOOK_EVENT: '' })).requests).toEqual([
`/hook/${source}`
])
}
})
it.skipIf(process.platform === 'win32')(
'drops Grok-replayed hooks without suppressing their protocol response',
() => {
for (const script of [
getClaudeManagedScript('posix', CLAUDE_SCRIPT_OPTIONS),
getCursorManagedScript('posix')
'keeps Grok-replayed hooks out of the spool',
async () => {
for (const [source, script] of [
['claude', getClaudeManagedScript('posix', CLAUDE_SCRIPT_OPTIONS)],
['cursor', getCursorManagedScript('posix')]
]) {
const result = runPosixHook(script, 'Stop')
expect(result.curlCalled).toBe(false)
expect(result.stdout).toBe('{}\n')
const ordinary = await runVendorHook(script, { GROK_HOOK_EVENT: '' }, 'spool')
expect(ordinary.requests).toEqual([])
expect(ordinary.spooled).toHaveLength(1)
expect(JSON.parse(ordinary.spooled[0])).toMatchObject({
source,
paneKey: 'tab:leaf',
payload: { hook_event_name: 'Stop' }
})
const replayed = await runVendorHook(script, { GROK_HOOK_EVENT: 'Stop' }, 'spool')
expect(replayed.requests).toEqual([])
expect(replayed.spooled).toEqual([])
}
}
)
it.skipIf(process.platform === 'win32')('leaves non-Grok hook delivery unchanged', () => {
for (const script of [
getClaudeManagedScript('posix', CLAUDE_SCRIPT_OPTIONS),
getCursorManagedScript('posix')
]) {
const result = runPosixHook(script, '')
expect(result.curlCalled).toBe(true)
expect(result.stdout).toBe('{}\n')
}
})
})
@@ -29,12 +29,14 @@ function claudeHook(hookEventName: string, promptId: string, extra: Record<strin
}
}
function postHook(port: number, token: string, payload: Record<string, unknown>) {
return fetch(`http://127.0.0.1:${port}/hook/claude`, {
async function postHook(port: number, token: string, payload: Record<string, unknown>) {
const response = await fetch(`http://127.0.0.1:${port}/hook/claude`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'X-Orca-Agent-Hook-Token': token },
body: JSON.stringify({ paneKey: PANE_KEY, payload })
})
expect(response.status).toBe(204)
return response
}
function turnEnvelope(): AgentHookRelayEnvelope {
@@ -1,4 +1,4 @@
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -14,10 +14,21 @@ const probe = vi.hoisted(() =>
)
vi.mock('../../shared/agent-process-presence-probe', () => ({ probeAgentProcessPresence: probe }))
const servers: AgentHookServer[] = []
const temporaryPaths: string[] = []
function createUserDataPath(prefix: string): string {
const path = mkdtempSync(join(tmpdir(), prefix))
temporaryPaths.push(path)
return path
}
afterEach(() => {
for (const server of servers.splice(0)) {
server.stop()
}
for (const path of temporaryPaths.splice(0)) {
rmSync(path, { recursive: true, force: true })
}
vi.restoreAllMocks()
probe.mockReset()
probe.mockResolvedValue('unverifiable')
})
@@ -257,7 +268,7 @@ describe('host-owned hook presence', () => {
})
it('checks each pane once after replaying its spooled hooks', async () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-presence-spool-'))
const userDataPath = createUserDataPath('orca-presence-spool-')
const first = new AgentHookServer()
servers.push(first)
await first.start({ env: 'production', userDataPath })
@@ -291,6 +302,10 @@ describe('host-owned hook presence', () => {
servers.push(restarted)
await restarted.start({ env: 'production', userDataPath })
expect(probe).toHaveBeenCalledOnce()
expect(restarted.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
toolName: 'Bash',
toolInput: 'ls'
})
})
it('keeps unanswered reads and clears only a positive process exit', async () => {
@@ -399,6 +414,7 @@ describe('retained agent exit', () => {
const server = await createServer()
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
expect(await server.checkAgentPresenceSince(PANE, Date.now() + 60_000)).toBeNull()
})
@@ -460,6 +476,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
await hook(server, 'UserPromptSubmit', 'replacement', undefined, null)
expect(await server.checkAgentPresenceSince(PANE, attachedAt)).toBeNull()
})
@@ -469,6 +486,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
server.ingestTerminalStatus({
paneKey: PANE,
tabId: 'tab-1',
@@ -528,6 +546,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
const response = await postHookEvent(
server,
buildBody({ hook_event_name: 'before_agent_start', prompt: 'next' }),
@@ -542,6 +561,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
const response = await postHookEvent(
server,
buildBody({ hook_event_name: 'agent_start', prompt: 'next' }),
@@ -552,7 +572,7 @@ describe('retained agent exit', () => {
})
it('keeps a persisted exit through a spooled start written before it', async () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-presence-exit-spool-'))
const userDataPath = createUserDataPath('orca-presence-exit-spool-')
const first = new AgentHookServer()
servers.push(first)
await first.start({ env: 'production', userDataPath })
@@ -586,13 +606,16 @@ describe('retained agent exit', () => {
})
it('lets a successor start spooled in the exit second clear the exit', async () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-presence-exit-same-second-'))
const userDataPath = createUserDataPath('orca-presence-exit-same-second-')
const first = new AgentHookServer()
servers.push(first)
await first.start({ env: 'production', userDataPath })
const attachedAt = Date.now() - 1
const exitedAt = Date.now()
vi.spyOn(Date, 'now').mockReturnValue(exitedAt)
const attachedAt = exitedAt - 1
await hook(first, 'SessionStart')
await hook(first, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await first.checkAgentPresenceSince(PANE, 0)).toBe('exited')
first.flushStatusPersistSync()
first.stop()
const spoolDir = join(userDataPath, 'agent-hooks', 'spool')
@@ -602,7 +625,7 @@ describe('retained agent exit', () => {
`\n${JSON.stringify({
paneKey: PANE,
source: 'pi',
receivedAt: Math.floor(Date.now() / 1000) * 1000,
receivedAt: Math.floor(exitedAt / 1000) * 1000,
payload: { hook_event_name: 'before_agent_start', prompt: 'next' }
})}\n`
)
@@ -617,6 +640,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
await hook(server, 'Stop', 'session-a', undefined, null)
probe.mockResolvedValue('live')
await hook(server, 'PreToolUse', 'replacement', undefined, 4002)
@@ -624,7 +648,7 @@ describe('retained agent exit', () => {
})
it('keeps the exit readable after the hook store restarts', async () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-presence-exit-'))
const userDataPath = createUserDataPath('orca-presence-exit-')
const first = new AgentHookServer()
servers.push(first)
await first.start({ env: 'production', userDataPath })
@@ -643,6 +667,7 @@ describe('retained agent exit', () => {
const server = await createServer()
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
server.forgetAgentExit(PANE)
expect(await server.checkAgentPresenceSince(PANE, 0)).toBeNull()
})
@@ -652,6 +677,7 @@ describe('retained agent exit', () => {
const attachedAt = Date.now() - 1
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
expect(await server.checkAgentPresenceSince(PANE, 0)).toBe('exited')
await hook(server, 'SessionStart', 'relaunch', undefined, 4002)
probe.mockResolvedValue('live')
expect(await server.checkAgentPresenceSince(PANE, attachedAt)).toBe('live')
@@ -701,7 +727,7 @@ describe('retained agent exit lifetime', () => {
it('prunes an exit older than the hydrate window when saving', async () => {
const server = new AgentHookServer()
servers.push(server)
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-presence-exit-prune-'))
const userDataPath = createUserDataPath('orca-presence-exit-prune-')
await server.start({ env: 'production', userDataPath })
await hook(server, 'SessionStart')
await hook(server, 'SessionEnd', 'session-a', 'prompt_input_exit')
@@ -0,0 +1,69 @@
import { createServer } from 'node:http'
import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { runProcess } from '@orca/process-host'
import { expect } from 'vitest'
export async function runVendorHook(
script: string,
hookEnv: NodeJS.ProcessEnv,
delivery: 'http' | 'spool' = 'http'
) {
const requests: string[] = []
const server = createServer((request, response) => {
requests.push(request.url ?? '')
request.resume()
request.on('end', () => response.end('{}'))
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const dir = mkdtempSync(join(tmpdir(), 'orca-vendor-hook-'))
try {
const address = server.address()
if (!address || typeof address === 'string') {
throw new Error('Expected a loopback port')
}
const windows = process.platform === 'win32'
const scriptPath = join(dir, windows ? 'hook.cmd' : 'hook.sh')
writeFileSync(scriptPath, script)
const endpointPath = join(dir, windows ? 'endpoint.cmd' : 'endpoint.env')
writeFileSync(endpointPath, '')
const result = await runProcess({
program: windows ? scriptPath : '/bin/sh',
args: windows ? [] : [scriptPath],
input: '{"hook_event_name":"Stop"}',
timeoutMs: 5_000,
env: {
...process.env,
CURSOR_VERSION: '',
CURSOR_PROJECT_DIR: '',
GROK_HOOK_EVENT: '',
DEVIN_SESSION_ID: '',
DEVIN_PROJECT_DIR: '',
ORCA_CURSOR_HOOK_RESPONSE: '',
ORCA_AGENT_HOOK_TRANSPORT: '',
CLAUDE_JOB_DIR: '',
ORCA_AGENT_HOOK_ENDPOINT: endpointPath,
ORCA_AGENT_HOOK_PORT: delivery === 'http' ? String(address.port) : '',
ORCA_AGENT_HOOK_TOKEN: 'test-token',
ORCA_PANE_KEY: 'tab:leaf',
...hookEnv
}
})
expect(result.code).toBe(0)
expect(result.timedOut).toBe(false)
expect(result.stdout.replaceAll('\r\n', '\n')).toBe('{}\n')
const spoolDir = join(dir, 'spool')
const spooled = existsSync(spoolDir)
? readdirSync(spoolDir).flatMap((file) =>
readFileSync(join(spoolDir, file), 'utf8').split('\n').filter(Boolean)
)
: []
return { requests, spooled }
} finally {
await new Promise<void>((resolve, reject) =>
server.close((error) => (error ? reject(error) : resolve()))
)
rmSync(dir, { recursive: true, force: true })
}
}